Compare commits

...

49 Commits

Author SHA1 Message Date
semantic-release 6e4a53cdfa chore(release): 0.2.11 [skip ci] 2025-02-14 04:25:18 +00:00
phernandez 2e215fe83c fix: ci installer app fix #37 2025-02-13 22:23:40 -06:00
semantic-release b546941779 chore(release): 0.2.10 [skip ci] 2025-02-14 04:17:24 +00:00
phernandez 29a259421a fix: fix build on github ci for app installer 2025-02-13 22:14:05 -06:00
semantic-release 071e76a0ff chore(release): 0.2.9 [skip ci] 2025-02-14 04:04:56 +00:00
phernandez 9ee67e9de0 Merge branch 'main' of github.com:basicmachines-co/basic-memory 2025-02-13 22:03:15 -06:00
phernandez edbc04be60 fix: fix installer on ci, maybe 2025-02-13 22:03:08 -06:00
semantic-release 864209c5f8 chore(release): 0.2.8 [skip ci] 2025-02-14 03:47:45 +00:00
phernandez b40f0c75da Merge branch 'main' of github.com:basicmachines-co/basic-memory 2025-02-13 21:46:07 -06:00
phernandez 230738ee9c fix: try to fix installer ci 2025-02-13 21:45:58 -06:00
semantic-release d15d8f91a4 chore(release): 0.2.7 [skip ci] 2025-02-14 03:40:31 +00:00
phernandez a6fdfad374 Merge branch 'main' of github.com:basicmachines-co/basic-memory 2025-02-13 21:33:45 -06:00
phernandez 3e78fcc2c2 fix: fix installer setup.py change ci to use make 2025-02-13 21:33:15 -06:00
semantic-release b5d09a4854 chore(release): 0.2.6 [skip ci] 2025-02-14 03:23:02 +00:00
phernandez 01d46727b4 fix: bump project patch version 2025-02-13 21:18:25 -06:00
phernandez 713b3bedf8 fix path to installer app in ci 2025-02-13 21:08:46 -06:00
semantic-release 97d3a0196e chore(release): 0.2.5 [skip ci] 2025-02-14 03:05:16 +00:00
phernandez 052f491fff fix: refix vitual env in installer build 2025-02-13 20:55:21 -06:00
semantic-release 6589f5d251 chore(release): 0.2.4 [skip ci] 2025-02-14 02:48:31 +00:00
phernandez 9a4070eccf Merge branch 'main' of github.com:basicmachines-co/basic-memory 2025-02-13 20:40:38 -06:00
phernandez 41d4d81c1a fix: workaround unsigned app 2025-02-13 20:40:33 -06:00
semantic-release 2bc4847c98 chore(release): 0.2.3 [skip ci] 2025-02-14 02:33:37 +00:00
phernandez 4b1be9e54f Merge branch 'main' of github.com:basicmachines-co/basic-memory 2025-02-13 20:26:20 -06:00
phernandez 53d220df58 fix: fix path to intaller app artifact 2025-02-13 20:24:46 -06:00
semantic-release 128b7657ec chore(release): 0.2.2 [skip ci] 2025-02-14 02:21:21 +00:00
phernandez 373640c843 Merge branch 'main' of github.com:basicmachines-co/basic-memory 2025-02-13 20:13:59 -06:00
phernandez d4c8293687 fix: activate vitualenv in installer build 2025-02-13 20:13:40 -06:00
semantic-release 0d48404af4 chore(release): 0.2.1 [skip ci] 2025-02-14 02:09:47 +00:00
phernandez f11bf78f3f fix: trigger installer build on release 2025-02-13 19:58:50 -06:00
semantic-release 31c55c7a32 chore(release): 0.2.0 [skip ci] 2025-02-14 01:49:04 +00:00
Paul Hernandez 7c381a59c9 feat: build installer via github action (#7)
* feat(ci): build installer via github action

* enforce conventional commits in PR titles

* feat: add icon to installer

---------

Co-authored-by: phernandez <phernandez@basicmachines.co>
2025-02-13 19:42:38 -06:00
semantic-release 40a8242002 chore(release): 0.1.2 [skip ci] 2025-02-14 00:22:35 +00:00
phernandez dde9ff228b fix: fix installer for mac 2025-02-13 18:15:22 -06:00
phernandez 2f9178b050 fix: update uv installer url 2025-02-13 17:54:52 -06:00
phernandez eb3360cc22 fix: remove unused FileChange dataclass 2025-02-13 17:50:59 -06:00
phernandez f1eeaee104 remove tray menu and rumps 2025-02-13 15:50:27 -06:00
phernandez 355519a786 refactor cli commands 2025-02-13 15:42:50 -06:00
phernandez 23f45ce788 add installer for mac/windows 2025-02-13 14:51:52 -06:00
phernandez 86380a92b2 log sync conflict at debug instead of info 2025-02-13 14:21:06 -06:00
phernandez a87ac58646 Merge branch 'installer' 2025-02-12 23:23:30 -06:00
phernandez 28513c8b8d lint code 2025-02-12 22:39:22 -06:00
Paul Hernandez c394d682e1 Alembic (#6)
* remove database service

* add alembic config for migrations

* fix type check and format

---------

Co-authored-by: phernandez <phernandez@basicmachines.co>
2025-02-12 22:12:52 -06:00
Paul Hernandez 5b4f0eafcc Pre release fixups (#5)
* configure logging

* set mcp output logging also

* fix type check errors

* fix type check

* rename Permalink schema type

* fix type errors

* add typechecks to ci workflow

* pytest coverage setup

* add tests for status cli

* sync tests coverage

* watch_service test coverage

* tests for tool_utils.py

* clean up imports

* file_utils coverage

* markdown plugins coverage

* 99% test coverage

* more test coverage, remove ObservationCategory

* more tool coverage

* fix type-check

* format, upgrade deps

---------

Co-authored-by: phernandez <phernandez@basicmachines.co>
2025-02-12 08:23:10 -06:00
phernandez 452308fd96 subprocess app tray wip 2025-02-08 12:35:28 -06:00
phernandez 8094528c40 menubar wip 2025-02-08 11:50:54 -06:00
phernandez fa77a41e6f menubar app wip 2025-02-08 10:22:47 -06:00
phernandez 12cd1e43b0 create .dmg 2025-02-07 23:09:48 -06:00
phernandez c58a2e0893 cx freeze installer 2025-02-07 22:02:59 -06:00
phernandez 28318d8435 add install.sh script 2025-02-07 20:32:06 -06:00
125 changed files with 3681 additions and 3808 deletions
+41
View File
@@ -0,0 +1,41 @@
name: "Pull Request Title"
on:
pull_request:
types:
- opened
- edited
- synchronize
jobs:
main:
runs-on: ubuntu-latest
steps:
- uses: amannn/action-semantic-pull-request@v5
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
# Configure allowed types based on what we want in our changelog
types: |
feat
fix
chore
docs
style
refactor
perf
test
build
ci
# Require at least one from scope list (optional)
scopes: |
core
cli
api
mcp
sync
ui
deps
installer
# Allow breaking changes (needs "!" after type/scope)
requireScopeForBreakingChange: true
+52 -1
View File
@@ -20,6 +20,9 @@ jobs:
permissions:
id-token: write
contents: write
outputs:
released: ${{ steps.release.outputs.released }}
tag: ${{ steps.release.outputs.tag }}
steps:
- uses: actions/checkout@v4
@@ -43,4 +46,52 @@ jobs:
if: steps.release.outputs.released == 'true'
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
tag: ${{ steps.release.outputs.tag }}
tag: ${{ steps.release.outputs.tag }}
build-macos:
needs: release
if: needs.release.outputs.released == 'true'
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.release.outputs.tag }}
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Install uv
run: |
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- name: Install librsvg
run: brew install librsvg
- name: Create icon
run: |
cd installer
chmod +x make_icons.sh
./make_icons.sh
- name: Build macOS installer
run: |
cd installer
uv venv
uv pip install -e "..[dev]"
uv run python setup.py bdist_mac
xattr -dr com.apple.quarantine "build/Basic Memory Installer.app"
- name: Zip macOS installer
run: |
cd installer/build
zip -r "Basic-Memory-Installer-${{ needs.release.outputs.tag }}.zip" "Basic Memory Installer.app"
- name: Upload macOS installer
uses: softprops/action-gh-release@v1
with:
files: installer/build/Basic-Memory-Installer-${{ needs.release.outputs.tag }}.zip
tag_name: ${{ needs.release.outputs.tag }}
token: ${{ secrets.GITHUB_TOKEN }}
+4
View File
@@ -37,6 +37,10 @@ jobs:
run: |
uv pip install -e .[dev]
- name: Run type checks
run: |
uv run make type-check
- name: Run tests
run: |
uv pip install pytest pytest-cov
+16 -14
View File
@@ -1,8 +1,10 @@
# Python
__pycache__/
*.py[cod]
*$py.class
*.so
__pycache__/
.pytest_cache/
.coverage
htmlcov/
# Distribution / packaging
.Python
build/
develop-eggs/
@@ -20,7 +22,12 @@ wheels/
.installed.cfg
*.egg
# Virtual Environment
# Installer artifacts
installer/build/
installer/dist/
rw.*.dmg # Temporary disk images
# Virtual environments
.env
.venv
env/
@@ -30,13 +37,8 @@ ENV/
# IDE
.idea/
.vscode/
*.swp
*.swo
# Project specific
projects/*.db
projects/*.db-journal
/.coverage
**/.DS_Store
*.log
/.coverage.*
# macOS
.DS_Store
+116
View File
@@ -1,6 +1,122 @@
# CHANGELOG
## v0.2.11 (2025-02-14)
### Bug Fixes
- Ci installer app fix #37
([`2e215fe`](https://github.com/basicmachines-co/basic-memory/commit/2e215fe83ca421b921186c7f1989dc2cb5cca278))
## v0.2.10 (2025-02-14)
### Bug Fixes
- Fix build on github ci for app installer
([`29a2594`](https://github.com/basicmachines-co/basic-memory/commit/29a259421a0ccb10cfa68e3707eaa506ad5e55c0))
## v0.2.9 (2025-02-14)
## v0.2.8 (2025-02-14)
### Bug Fixes
- Fix installer on ci, maybe
([`edbc04b`](https://github.com/basicmachines-co/basic-memory/commit/edbc04be601d234bb1f5eb3ba24d6ad55244b031))
## v0.2.7 (2025-02-14)
### Bug Fixes
- Try to fix installer ci
([`230738e`](https://github.com/basicmachines-co/basic-memory/commit/230738ee9c110c0509e0a09cb0e101a92cfcb729))
## v0.2.6 (2025-02-14)
### Bug Fixes
- Bump project patch version
([`01d4672`](https://github.com/basicmachines-co/basic-memory/commit/01d46727b40c24b017ea9db4b741daef565ac73e))
- Fix installer setup.py change ci to use make
([`3e78fcc`](https://github.com/basicmachines-co/basic-memory/commit/3e78fcc2c208d83467fe7199be17174d7ffcad1a))
## v0.2.5 (2025-02-14)
### Bug Fixes
- Refix vitual env in installer build
([`052f491`](https://github.com/basicmachines-co/basic-memory/commit/052f491fff629e8ead629c9259f8cb46c608d584))
## v0.2.4 (2025-02-14)
## v0.2.3 (2025-02-14)
### Bug Fixes
- Workaround unsigned app
([`41d4d81`](https://github.com/basicmachines-co/basic-memory/commit/41d4d81c1ad1dc2923ba0e903a57454a0c8b6b5c))
## v0.2.2 (2025-02-14)
### Bug Fixes
- Fix path to intaller app artifact
([`53d220d`](https://github.com/basicmachines-co/basic-memory/commit/53d220df585561f9edd0d49a9e88f1d4055059cf))
## v0.2.1 (2025-02-14)
### Bug Fixes
- Activate vitualenv in installer build
([`d4c8293`](https://github.com/basicmachines-co/basic-memory/commit/d4c8293687a52eaf3337fe02e2f7b80e4cc9a1bb))
- Trigger installer build on release
([`f11bf78`](https://github.com/basicmachines-co/basic-memory/commit/f11bf78f3f600d0e1b01996cf8e1f9c39e3dd218))
## v0.2.0 (2025-02-14)
### Features
- Build installer via github action ([#7](https://github.com/basicmachines-co/basic-memory/pull/7),
[`7c381a5`](https://github.com/basicmachines-co/basic-memory/commit/7c381a59c962053c78da096172e484f28ab47e96))
* feat(ci): build installer via github action
* enforce conventional commits in PR titles
* feat: add icon to installer
---------
Co-authored-by: phernandez <phernandez@basicmachines.co>
## v0.1.2 (2025-02-14)
### Bug Fixes
- Fix installer for mac
([`dde9ff2`](https://github.com/basicmachines-co/basic-memory/commit/dde9ff228b72852b5abc58faa1b5e7c6f8d2c477))
- Remove unused FileChange dataclass
([`eb3360c`](https://github.com/basicmachines-co/basic-memory/commit/eb3360cc221f892b12a17137ae740819d48248e8))
- Update uv installer url
([`2f9178b`](https://github.com/basicmachines-co/basic-memory/commit/2f9178b0507b3b69207d5c80799f2d2f573c9a04))
## v0.1.1 (2025-02-07)
+23 -20
View File
@@ -1,39 +1,42 @@
.PHONY: install test lint db-new db-up db-down db-reset
.PHONY: install test lint clean format type-check installer-mac installer-win
install:
brew install dbmate
pip install -e ".[dev]"
test:
pytest -p pytest_mock -v
lint:
black .
ruff check .
ruff check . --fix
db-new:
dbmate new $(name)
db-up:
dbmate up
db-down:
dbmate down
db-reset:
dbmate drop
dbmate up
type-check:
uv run pyright
clean:
find . -type f -name '*.pyc' -delete
find . -type d -name '__pycache__' -exec rm -r {} +
rm -rf installer/build/
rm -rf installer/dist/
rm -f rw.*.dmg
rm -rf dist
rm -rf installer/build
rm -rf installer/dist
rm -f .coverage.*
format:
uv run ruff format .
format: format-python
#format: format-python format-prettier
# run inspector tool
run-dev:
uv run mcp dev src/basic_memory/mcp/main.py
uv run mcp dev src/basic_memory/mcp/main.py
# Build app installer
installer-mac:
cd installer && uv run python setup.py bdist_mac
installer-win:
cd installer && uv run python setup.py bdist_win32
update-deps:
uv lock f--upgrade
+119
View File
@@ -0,0 +1,119 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
# Use forward slashes (/) also on windows to provide an os agnostic path
script_location = src/basic_memory/alembic
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
# for all available tokens
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
# sys.path path, will be prepended to sys.path if present.
# defaults to the current working directory.
prepend_sys_path = .
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
# string value is passed to ZoneInfo()
# leave blank for localtime
# timezone =
# max length of characters to apply to the "slug" field
# truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
# set to 'true' to allow .pyc and .pyo files without
# a source .py file to be detected as revisions in the
# versions/ directory
# sourceless = false
# version location specification; This defaults
# to migrations/versions. When using multiple version
# directories, initial revisions must be specified with --version-path.
# The path separator used here should be the separator specified by "version_path_separator" below.
# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
# version path separator; As mentioned above, this is the character used to split
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
# Valid values for version_path_separator are:
#
# version_path_separator = :
# version_path_separator = ;
# version_path_separator = space
# version_path_separator = newline
#
# Use os.pathsep. Default configuration used for new projects.
version_path_separator = os
# set to 'true' to search source files recursively
# in each "version_locations" directory
# new in Alembic version 1.10
# recursive_version_locations = false
# the output encoding used when revision files
# are written from script.py.mako
# output_encoding = utf-8
sqlalchemy.url = driver://user:pass@localhost/dbname
[post_write_hooks]
# post_write_hooks defines scripts or Python functions that are run
# on newly generated revision scripts. See the documentation for further
# detail and examples
# format using "black" - use the console_scripts runner, against the "black" entrypoint
# hooks = black
# black.type = console_scripts
# black.entrypoint = black
# black.options = -l 79 REVISION_SCRIPT_FILENAME
# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
# hooks = ruff
# ruff.type = exec
# ruff.executable = %(here)s/.venv/bin/ruff
# ruff.options = --fix REVISION_SCRIPT_FILENAME
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARNING
handlers = console
qualname =
[logger_sqlalchemy]
level = WARNING
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
Binary file not shown.
+16
View File
@@ -0,0 +1,16 @@
# Basic Memory Installer
This installer configures Basic Memory to work with Claude Desktop.
## Installation
1. Download the latest installer from the [releases page](https://github.com/basicmachines-co/basic-memory/releases)
2. Unzip the downloaded file
3. Since the app is currently unsigned, you'll need to:
- Right-click (or Control-click) the app
- Select "Open" from the context menu
- Click "Open" in the warning dialog
4. Follow the installation instructions
5. Restart Claude Desktop
The warning only appears the first time you open the app. Future updates will include proper code signing.
+64
View File
@@ -0,0 +1,64 @@
<?xml version="1.0" encoding="UTF-8"?>
<svg viewBox="0 0 512 512" xmlns="http://www.w3.org/2000/svg">
<!-- Background -->
<rect x="0" y="0" width="512" height="512" rx="64" fill="#111111"/>
<!-- Define arrowhead marker -->
<defs>
<marker id="arrowhead"
markerWidth="10"
markerHeight="10"
refX="8"
refY="5"
orient="auto">
<path d="M 0 0 L 10 5 L 0 10 Z"
fill="#00cc00"/>
</marker>
</defs>
<!-- State 1 (initial) -->
<circle cx="156" cy="256" r="30" fill="none" stroke="#00cc00" stroke-width="3"/>
<!-- State 2 (accept) -->
<circle cx="356" cy="176" r="34" fill="none" stroke="#00cc00" stroke-width="3"/>
<circle cx="356" cy="176" r="28" fill="none" stroke="#00cc00" stroke-width="3"/>
<!-- State 3 (accept) -->
<circle cx="356" cy="336" r="34" fill="none" stroke="#00cc00" stroke-width="3"/>
<circle cx="356" cy="336" r="28" fill="none" stroke="#00cc00" stroke-width="3"/>
<!-- Initial arrow -->
<path d="M 96 256 L 126 256"
stroke="#00cc00" stroke-width="3" fill="none"
marker-end="url(#arrowhead)"/>
<!-- State transitions -->
<!-- 1 -> 2 -->
<path d="M 180 240
Q 260 200, 320 176"
stroke="#00cc00" stroke-width="3" fill="none"
marker-end="url(#arrowhead)"/>
<!-- 1 -> 3 -->
<path d="M 180 272
Q 260 312, 320 336"
stroke="#00cc00" stroke-width="3" fill="none"
marker-end="url(#arrowhead)"/>
<!-- Self loops -->
<path d="M 356 142
Q 396 142, 396 176
Q 396 210, 356 210
Q 316 210, 316 176
Q 316 142, 356 142"
stroke="#00cc00" stroke-width="2" fill="none"
marker-end="url(#arrowhead)"/>
<path d="M 356 302
Q 396 302, 396 336
Q 396 370, 356 370
Q 316 370, 316 336
Q 316 302, 356 302"
stroke="#00cc00" stroke-width="2" fill="none"
marker-end="url(#arrowhead)"/>
</svg>

After

Width:  |  Height:  |  Size: 2.0 KiB

+89
View File
@@ -0,0 +1,89 @@
import json
import subprocess
import sys
from pathlib import Path
# Use tkinter for GUI alerts on macOS
if sys.platform == "darwin":
import tkinter as tk
from tkinter import messagebox
def ensure_uv_installed():
"""Check if uv is installed, install if not."""
try:
subprocess.run(["uv", "--version"], capture_output=True, check=True)
except (subprocess.CalledProcessError, FileNotFoundError):
print("Installing uv package manager...")
subprocess.run(
[
"curl",
"-LsSf",
"https://astral.sh/uv/install.sh",
"|",
"sh",
],
shell=True,
)
def get_config_path():
"""Get Claude Desktop config path for current platform."""
if sys.platform == "darwin":
return Path.home() / "Library/Application Support/Claude/claude_desktop_config.json"
elif sys.platform == "win32":
return Path.home() / "AppData/Roaming/Claude/claude_desktop_config.json"
else:
raise RuntimeError(f"Unsupported platform: {sys.platform}")
def update_claude_config():
"""Update Claude Desktop config to include basic-memory."""
config_path = get_config_path()
config_path.parent.mkdir(parents=True, exist_ok=True)
# Load existing config or create new
if config_path.exists():
config = json.loads(config_path.read_text())
else:
config = {"mcpServers": {}}
# Add/update basic-memory config
config["mcpServers"]["basic-memory"] = {"command": "uvx", "args": ["basic-memory", "mcp"]}
# Write back config
config_path.write_text(json.dumps(config, indent=2))
def print_completion_message():
"""Show completion message with helpful tips."""
message = """Installation complete! Basic Memory is now available in Claude Desktop.
Please restart Claude Desktop for changes to take effect.
Quick Start:
1. You can run sync directly using: uvx basic-memory sync
2. Optionally, install globally with: uv pip install basic-memory
Built with ♥️ by Basic Machines."""
if sys.platform == "darwin":
# Show GUI message on macOS
root = tk.Tk()
root.withdraw() # Hide the main window
messagebox.showinfo("Basic Memory", message)
root.destroy()
else:
# Fallback to console output
print(message)
def main():
print("Welcome to Basic Memory installer")
ensure_uv_installed()
print("Configuring Claude Desktop...")
update_claude_config()
print_completion_message()
if __name__ == "__main__":
main()
+27
View File
@@ -0,0 +1,27 @@
#!/bin/bash
# Convert SVG to PNG at various required sizes
rsvg-convert -h 16 -w 16 icon.svg > icon_16x16.png
rsvg-convert -h 32 -w 32 icon.svg > icon_32x32.png
rsvg-convert -h 128 -w 128 icon.svg > icon_128x128.png
rsvg-convert -h 256 -w 256 icon.svg > icon_256x256.png
rsvg-convert -h 512 -w 512 icon.svg > icon_512x512.png
# Create iconset directory
mkdir -p Basic.iconset
# Move files into iconset with Mac-specific names
cp icon_16x16.png Basic.iconset/icon_16x16.png
cp icon_32x32.png Basic.iconset/icon_16x16@2x.png
cp icon_32x32.png Basic.iconset/icon_32x32.png
cp icon_128x128.png Basic.iconset/icon_32x32@2x.png
cp icon_256x256.png Basic.iconset/icon_128x128.png
cp icon_512x512.png Basic.iconset/icon_256x256.png
cp icon_512x512.png Basic.iconset/icon_512x512.png
# Convert iconset to icns
iconutil -c icns Basic.iconset
# Clean up
rm -rf Basic.iconset
rm icon_*.png
+50
View File
@@ -0,0 +1,50 @@
from cx_Freeze import setup, Executable
import sys
# Build options for all platforms - keep it simple and don't exclude too much
build_exe_options = {
"packages": ["json", "pathlib"],
"excludes": [
"unittest",
"pydoc",
"test"
],
# Control file copying behavior
"copy_dependent_files": False,
"create_library_zip": True,
"copy_to_same_dir": False,
}
# Platform-specific options
if sys.platform == "win32":
base = "Win32GUI" # Use GUI base for Windows
build_exe_options.update({
"include_msvcr": True, # Include Visual C++ runtime
})
target_name = "Basic Memory Installer.exe"
else: # darwin
base = None # Don't use GUI base for macOS
target_name = "Basic Memory Installer"
executables = [
Executable(
script="installer.py",
target_name=target_name,
base=base,
icon="Basic.icns"
)
]
setup(
name="basic-memory",
version=open("../pyproject.toml").read().split('version = "', 1)[1].split('"', 1)[0],
description="Basic Memory - Local-first knowledge management",
options={
"build_exe": build_exe_options,
"bdist_mac": {
"bundle_name": "Basic Memory Installer",
"iconfile": "Basic.icns"
}
},
executables=executables,
)
+17 -10
View File
@@ -1,6 +1,6 @@
[project]
name = "basic-memory"
version = "0.1.1"
version = "0.2.11"
description = "Local-first knowledge management combining Zettelkasten with knowledge graphs"
readme = "README.md"
requires-python = ">=3.12.1"
@@ -28,16 +28,9 @@ dependencies = [
"watchfiles>=1.0.4",
"fastapi[standard]>=0.115.8",
"alembic>=1.14.1",
"qasync>=0.27.1",
]
[project.optional-dependencies]
dev = [
"pytest>=8.3.4",
"pytest-cov>=4.1.0",
"pytest-mock>=3.12.0",
"pytest-asyncio>=0.24.0",
"ruff>=0.1.6",
]
[project.urls]
Homepage = "https://github.com/basicmachines-co/basic-memory"
@@ -64,10 +57,22 @@ target-version = "py312"
[tool.uv]
dev-dependencies = [
"gevent>=24.11.1",
"icecream>=2.1.3",
"pytest>=8.3.4",
"pytest-cov>=4.1.0",
"pytest-mock>=3.12.0",
"pytest-asyncio>=0.24.0",
"ruff>=0.1.6",
"pytest>=8.3.4",
"pytest-cov>=4.1.0",
"pytest-mock>=3.12.0",
"pytest-asyncio>=0.24.0",
"ruff>=0.1.6",
"cx-freeze>=7.2.10",
"pyqt6>=6.8.1",
]
[tool.pyright]
include = ["src/"]
exclude = ["**/__pycache__"]
@@ -91,3 +96,5 @@ dist_path = "dist/"
upload_to_pypi = true
commit_message = "chore(release): {version} [skip ci]"
[tool.coverage.run]
concurrency = ["thread", "gevent"]
+36
View File
@@ -0,0 +1,36 @@
#!/bin/bash
set -e
echo "Welcome to Basic Memory installer"
# 1. Install uv if not present
if ! command -v uv &> /dev/null; then
echo "Installing uv package manager..."
curl -LsSf https://github.com/astral-sh/uv/releases/download/0.1.23/uv-installer.sh | sh
fi
# 2. Configure Claude Desktop
echo "Configuring Claude Desktop..."
CONFIG_FILE="$HOME/Library/Application Support/Claude/claude_desktop_config.json"
# Create config directory if it doesn't exist
mkdir -p "$(dirname "$CONFIG_FILE")"
# If config file doesn't exist, create it with initial structure
if [ ! -f "$CONFIG_FILE" ]; then
echo '{"mcpServers": {}}' > "$CONFIG_FILE"
fi
# Add/update the basic-memory config using jq
jq '.mcpServers."basic-memory" = {
"command": "uvx",
"args": ["basic-memory"]
}' "$CONFIG_FILE" > "$CONFIG_FILE.tmp" && mv "$CONFIG_FILE.tmp" "$CONFIG_FILE"
echo "Installation complete! Basic Memory is now available in Claude Desktop."
echo "Please restart Claude Desktop for changes to take effect."
echo -e "\nQuick Start:"
echo "1. You can run sync directly using: uvx basic-memory sync"
echo "2. Optionally, install globally with: uv pip install basic-memory"
echo -e "\nBuilt with ♥️ by Basic Machines."
+1 -1
View File
@@ -1,3 +1,3 @@
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
__version__ = "0.0.1"
__version__ = "0.0.1"
+1
View File
@@ -0,0 +1 @@
Generic single-database configuration.
+75
View File
@@ -0,0 +1,75 @@
"""Alembic environment configuration."""
from logging.config import fileConfig
from sqlalchemy import engine_from_config
from sqlalchemy import pool
from alembic import context
from basic_memory.models import Base
from basic_memory.config import config as app_config
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Set the SQLAlchemy URL from our app config
sqlalchemy_url = f"sqlite:///{app_config.database_path}"
config.set_main_option("sqlalchemy.url", sqlalchemy_url)
# Interpret the config file for Python logging.
if config.config_file_name is not None:
fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
target_metadata = Base.metadata
def run_migrations_offline() -> None:
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
+29
View File
@@ -0,0 +1,29 @@
"""Functions for managing database migrations."""
import asyncio
from pathlib import Path
from loguru import logger
from alembic.config import Config
from alembic import command
def get_alembic_config() -> Config: # pragma: no cover
"""Get alembic config with correct paths."""
migrations_path = Path(__file__).parent
alembic_ini = migrations_path.parent.parent.parent / "alembic.ini"
config = Config(alembic_ini)
config.set_main_option("script_location", str(migrations_path))
return config
async def reset_database(): # pragma: no cover
"""Drop and recreate all tables."""
logger.info("Resetting database...")
config = get_alembic_config()
def _reset(cfg):
command.downgrade(cfg, "base")
command.upgrade(cfg, "head")
await asyncio.get_event_loop().run_in_executor(None, _reset, config)
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,93 @@
"""initial schema
Revision ID: 3dae7c7b1564
Revises:
Create Date: 2025-02-12 21:23:00.336344
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "3dae7c7b1564"
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table(
"entity",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("title", sa.String(), nullable=False),
sa.Column("entity_type", sa.String(), nullable=False),
sa.Column("entity_metadata", sa.JSON(), nullable=True),
sa.Column("content_type", sa.String(), nullable=False),
sa.Column("permalink", sa.String(), nullable=False),
sa.Column("file_path", sa.String(), nullable=False),
sa.Column("checksum", sa.String(), nullable=True),
sa.Column("created_at", sa.DateTime(), nullable=False),
sa.Column("updated_at", sa.DateTime(), nullable=False),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("permalink", name="uix_entity_permalink"),
)
op.create_index("ix_entity_created_at", "entity", ["created_at"], unique=False)
op.create_index(op.f("ix_entity_file_path"), "entity", ["file_path"], unique=True)
op.create_index(op.f("ix_entity_permalink"), "entity", ["permalink"], unique=True)
op.create_index("ix_entity_title", "entity", ["title"], unique=False)
op.create_index("ix_entity_type", "entity", ["entity_type"], unique=False)
op.create_index("ix_entity_updated_at", "entity", ["updated_at"], unique=False)
op.create_table(
"observation",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("entity_id", sa.Integer(), nullable=False),
sa.Column("content", sa.Text(), nullable=False),
sa.Column("category", sa.String(), nullable=False),
sa.Column("context", sa.Text(), nullable=True),
sa.Column("tags", sa.JSON(), server_default="[]", nullable=True),
sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
)
op.create_index("ix_observation_category", "observation", ["category"], unique=False)
op.create_index("ix_observation_entity_id", "observation", ["entity_id"], unique=False)
op.create_table(
"relation",
sa.Column("id", sa.Integer(), nullable=False),
sa.Column("from_id", sa.Integer(), nullable=False),
sa.Column("to_id", sa.Integer(), nullable=True),
sa.Column("to_name", sa.String(), nullable=False),
sa.Column("relation_type", sa.String(), nullable=False),
sa.Column("context", sa.Text(), nullable=True),
sa.ForeignKeyConstraint(["from_id"], ["entity.id"], ondelete="CASCADE"),
sa.ForeignKeyConstraint(["to_id"], ["entity.id"], ondelete="CASCADE"),
sa.PrimaryKeyConstraint("id"),
sa.UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation"),
)
op.create_index("ix_relation_from_id", "relation", ["from_id"], unique=False)
op.create_index("ix_relation_to_id", "relation", ["to_id"], unique=False)
op.create_index("ix_relation_type", "relation", ["relation_type"], unique=False)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_index("ix_relation_type", table_name="relation")
op.drop_index("ix_relation_to_id", table_name="relation")
op.drop_index("ix_relation_from_id", table_name="relation")
op.drop_table("relation")
op.drop_index("ix_observation_entity_id", table_name="observation")
op.drop_index("ix_observation_category", table_name="observation")
op.drop_table("observation")
op.drop_index("ix_entity_updated_at", table_name="entity")
op.drop_index("ix_entity_type", table_name="entity")
op.drop_index("ix_entity_title", table_name="entity")
op.drop_index(op.f("ix_entity_permalink"), table_name="entity")
op.drop_index(op.f("ix_entity_file_path"), table_name="entity")
op.drop_index("ix_entity_created_at", table_name="entity")
op.drop_table("entity")
# ### end Alembic commands ###
+2 -1
View File
@@ -1,4 +1,5 @@
"""Basic Memory API module."""
from .app import app
__all__ = ["app"]
__all__ = ["app"]
+26 -24
View File
@@ -7,40 +7,42 @@ from fastapi.exception_handlers import http_exception_handler
from loguru import logger
from basic_memory import db
from basic_memory.config import config as app_config
from basic_memory.api.routers import knowledge, search, memory, resource
from basic_memory.config import config
from basic_memory.services import DatabaseService
from alembic import command
from alembic.config import Config
from basic_memory.db import DatabaseType
from basic_memory.repository.search_repository import SearchRepository
async def run_migrations(): # pragma: no cover
"""Run any pending alembic migrations."""
logger.info("Running database migrations...")
try:
config = Config("alembic.ini")
command.upgrade(config, "head")
logger.info("Migrations completed successfully")
_, session_maker = await db.get_or_create_db(
app_config.database_path, DatabaseType.FILESYSTEM
)
await SearchRepository(session_maker).init_search_index()
except Exception as e:
logger.error(f"Error running migrations: {e}")
raise
@asynccontextmanager
async def lifespan(app: FastAPI):
async def lifespan(app: FastAPI): # pragma: no cover
"""Lifecycle manager for the FastAPI app."""
logger.info("Starting Basic Memory API")
# check the db state
await check_db(app)
await run_migrations()
yield
logger.info("Shutting down Basic Memory API")
await db.shutdown_db()
async def check_db(app: FastAPI):
logger.info("Checking database state")
# Initialize DB management service
db_service = DatabaseService(
config=config,
)
# Check and initialize DB if needed
if not await db_service.check_db():
raise RuntimeError("Database initialization failed")
# Clean up old backups on shutdown
await db_service.cleanup_backups()
# Initialize FastAPI app
app = FastAPI(
title="Basic Memory API",
@@ -57,7 +59,7 @@ app.include_router(resource.router)
@app.exception_handler(Exception)
async def exception_handler(request, exc):
async def exception_handler(request, exc): # pragma: no cover
logger.exception(
f"An unhandled exception occurred for request '{request.url}', exception: {exc}"
)
@@ -17,7 +17,7 @@ from basic_memory.schemas import (
DeleteEntitiesResponse,
DeleteEntitiesRequest,
)
from basic_memory.schemas.base import PathId, Entity
from basic_memory.schemas.base import Permalink, Entity
from basic_memory.services.exceptions import EntityNotFoundError
router = APIRouter(prefix="/knowledge", tags=["knowledge"])
@@ -27,10 +27,10 @@ router = APIRouter(prefix="/knowledge", tags=["knowledge"])
@router.post("/entities", response_model=EntityResponse)
async def create_entity(
data: Entity,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
data: Entity,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
) -> EntityResponse:
"""Create an entity."""
logger.info(f"request: create_entity with data={data}")
@@ -47,12 +47,12 @@ async def create_entity(
@router.put("/entities/{permalink:path}", response_model=EntityResponse)
async def create_or_update_entity(
permalink: PathId,
data: Entity,
response: Response,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
permalink: Permalink,
data: Entity,
response: Response,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service: SearchServiceDep,
) -> EntityResponse:
"""Create or update an entity. If entity exists, it will be updated, otherwise created."""
logger.info(f"request: create_or_update_entity with permalink={permalink}, data={data}")
@@ -69,7 +69,9 @@ async def create_or_update_entity(
await search_service.index_entity(entity, background_tasks=background_tasks)
result = EntityResponse.model_validate(entity)
logger.info(f"response: create_or_update_entity with result={result}, status_code={response.status_code}")
logger.info(
f"response: create_or_update_entity with result={result}, status_code={response.status_code}"
)
return result
@@ -78,8 +80,8 @@ async def create_or_update_entity(
@router.get("/entities/{permalink:path}", response_model=EntityResponse)
async def get_entity(
entity_service: EntityServiceDep,
permalink: str,
entity_service: EntityServiceDep,
permalink: str,
) -> EntityResponse:
"""Get a specific entity by ID.
@@ -102,13 +104,13 @@ async def get_entity(
@router.get("/entities", response_model=EntityListResponse)
async def get_entities(
entity_service: EntityServiceDep,
permalink: Annotated[list[str] | None, Query()] = None,
entity_service: EntityServiceDep,
permalink: Annotated[list[str] | None, Query()] = None,
) -> EntityListResponse:
"""Open specific entities"""
logger.info(f"request: get_entities with permalinks={permalink}")
entities = await entity_service.get_entities_by_permalinks(permalink)
entities = await entity_service.get_entities_by_permalinks(permalink) if permalink else []
result = EntityListResponse(
entities=[EntityResponse.model_validate(entity) for entity in entities]
)
@@ -122,11 +124,11 @@ async def get_entities(
@router.delete("/entities/{identifier:path}", response_model=DeleteEntitiesResponse)
async def delete_entity(
identifier: str,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
link_resolver: LinkResolverDep,
search_service=Depends(get_search_service),
identifier: str,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
link_resolver: LinkResolverDep,
search_service=Depends(get_search_service),
) -> DeleteEntitiesResponse:
"""Delete a single entity and remove from search index."""
logger.info(f"request: delete_entity with identifier={identifier}")
@@ -149,10 +151,10 @@ async def delete_entity(
@router.post("/entities/delete", response_model=DeleteEntitiesResponse)
async def delete_entities(
data: DeleteEntitiesRequest,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service=Depends(get_search_service),
data: DeleteEntitiesRequest,
background_tasks: BackgroundTasks,
entity_service: EntityServiceDep,
search_service=Depends(get_search_service),
) -> DeleteEntitiesResponse:
"""Delete entities and remove from search index."""
logger.info(f"request: delete_entities with data={data}")
+17 -11
View File
@@ -1,13 +1,11 @@
"""Routes for memory:// URI operations."""
from datetime import datetime, timedelta
from typing import Optional, List, Annotated
from typing import Annotated
from dateparser import parse
from fastapi import APIRouter, Query
from loguru import logger
from basic_memory.config import config
from basic_memory.deps import ContextServiceDep, EntityRepositoryDep
from basic_memory.repository import EntityRepository
from basic_memory.repository.search_repository import SearchIndexRow
@@ -17,7 +15,8 @@ from basic_memory.schemas.memory import (
RelationSummary,
EntitySummary,
ObservationSummary,
MemoryMetadata, normalize_memory_url,
MemoryMetadata,
normalize_memory_url,
)
from basic_memory.schemas.search import SearchItemType
from basic_memory.services.context_service import ContextResultRow
@@ -25,12 +24,14 @@ from basic_memory.services.context_service import ContextResultRow
router = APIRouter(prefix="/memory", tags=["memory"])
async def to_graph_context(context, entity_repository: EntityRepository):
# return results
async def to_summary(item: SearchIndexRow | ContextResultRow):
match item.type:
case SearchItemType.ENTITY:
assert item.title is not None
assert item.created_at is not None
return EntitySummary(
title=item.title,
permalink=item.permalink,
@@ -38,12 +39,18 @@ async def to_graph_context(context, entity_repository: EntityRepository):
created_at=item.created_at,
)
case SearchItemType.OBSERVATION:
assert item.category is not None
assert item.content is not None
return ObservationSummary(
category=item.category, content=item.content, permalink=item.permalink
)
case SearchItemType.RELATION:
assert item.from_id is not None
from_entity = await entity_repository.find_by_id(item.from_id)
to_entity = await entity_repository.find_by_id(item.to_id)
assert from_entity is not None
to_entity = await entity_repository.find_by_id(item.to_id) if item.to_id else None
return RelationSummary(
permalink=item.permalink,
@@ -51,6 +58,8 @@ async def to_graph_context(context, entity_repository: EntityRepository):
from_id=from_entity.permalink,
to_id=to_entity.permalink if to_entity else None,
)
case _: # pragma: no cover
raise ValueError(f"Unexpected type: {item.type}")
primary_results = [await to_summary(r) for r in context["primary_results"]]
related_results = [await to_summary(r) for r in context["related_results"]]
@@ -61,7 +70,6 @@ async def to_graph_context(context, entity_repository: EntityRepository):
)
@router.get("/recent", response_model=GraphContext)
async def recent(
context_service: ContextServiceDep,
@@ -91,7 +99,8 @@ async def recent(
return await to_graph_context(context, entity_repository=entity_repository)
# get_memory_context needs to be declared last so other paths can match
# get_memory_context needs to be declared last so other paths can match
@router.get("/{uri:path}", response_model=GraphContext)
async def get_memory_context(
@@ -118,6 +127,3 @@ async def get_memory_context(
memory_url, depth=depth, since=since, max_results=max_results
)
return await to_graph_context(context, entity_repository=entity_repository)
+6 -12
View File
@@ -1,34 +1,28 @@
"""Router for search operations."""
from dataclasses import asdict
from fastapi import APIRouter, Depends, BackgroundTasks
from typing import List
from loguru import logger
from basic_memory.services.search_service import SearchService
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchResponse
from basic_memory.deps import get_search_service
router = APIRouter(prefix="/search", tags=["search"])
@router.post("/", response_model=SearchResponse)
async def search(
query: SearchQuery,
search_service: SearchService = Depends(get_search_service)
):
async def search(query: SearchQuery, search_service: SearchService = Depends(get_search_service)):
"""Search across all knowledge and documents."""
results = await search_service.search(query)
search_results = [SearchResult.model_validate(asdict(r)) for r in results]
return SearchResponse(results=search_results)
@router.post("/reindex")
async def reindex(
background_tasks: BackgroundTasks,
search_service: SearchService = Depends(get_search_service)
background_tasks: BackgroundTasks, search_service: SearchService = Depends(get_search_service)
):
"""Recreate and populate the search index."""
await search_service.reindex_all(background_tasks=background_tasks)
return {
"status": "ok",
"message": "Reindex initiated"
}
return {"status": "ok", "message": "Reindex initiated"}
+1 -1
View File
@@ -1 +1 @@
"""CLI tools for basic-memory"""
"""CLI tools for basic-memory"""
-1
View File
@@ -1,4 +1,3 @@
import typer
app = typer.Typer()
+3 -3
View File
@@ -1,5 +1,5 @@
"""Command module exports."""
"""CLI commands for basic-memory."""
from . import status, sync, import_memory_json
from . import status, sync, db, import_memory_json, mcp
__all__ = [ "status", "sync", "import_memory_json.py"]
__all__ = ["status", "sync", "db", "import_memory_json", "mcp"]
+25
View File
@@ -0,0 +1,25 @@
"""Database management commands."""
import asyncio
import typer
from loguru import logger
from basic_memory.alembic import migrations
from basic_memory.cli.app import app
@app.command()
def reset(
reindex: bool = typer.Option(False, "--reindex", help="Rebuild indices from filesystem"),
): # pragma: no cover
"""Reset database (drop all tables and recreate)."""
if typer.confirm("This will delete all data. Are you sure?"):
logger.info("Resetting database...")
asyncio.run(migrations.reset_database())
if reindex:
# Import and run sync
from basic_memory.cli.commands.sync import sync
logger.info("Rebuilding search index from filesystem...")
asyncio.run(sync()) # pyright: ignore
@@ -19,13 +19,16 @@ from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Obs
console = Console()
async def process_memory_json(json_path: Path, base_path: Path,markdown_processor: MarkdownProcessor):
async def process_memory_json(
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
):
"""Import entities from memory.json using markdown processor."""
# First pass - collect all relations by source entity
entity_relations: Dict[str, List[Relation]] = {}
entities: Dict[str, Dict[str, Any]] = {}
with Progress(
SpinnerColumn(),
TextColumn("[progress.description]{task.description}"),
@@ -34,12 +37,12 @@ async def process_memory_json(json_path: Path, base_path: Path,markdown_processo
console=console,
) as progress:
read_task = progress.add_task("Reading memory.json...", total=None)
# First pass - collect entities and relations
with open(json_path) as f:
lines = f.readlines()
progress.update(read_task, total=len(lines))
for line in lines:
data = json.loads(line)
if data["type"] == "entity":
@@ -52,14 +55,14 @@ async def process_memory_json(json_path: Path, base_path: Path,markdown_processo
entity_relations[source].append(
Relation(
type=data.get("relationType") or data.get("relation_type"),
target=data.get("to") or data.get("to_id")
target=data.get("to") or data.get("to_id"),
)
)
progress.update(read_task, advance=1)
# Second pass - create and write entities
write_task = progress.add_task("Creating entities...", total=len(entities))
entities_created = 0
for name, entity_data in entities.items():
entity = EntityMarkdown(
@@ -67,26 +70,25 @@ async def process_memory_json(json_path: Path, base_path: Path,markdown_processo
metadata={
"type": entity_data["entityType"],
"title": name,
"permalink": f"{entity_data['entityType']}/{name}"
"permalink": f"{entity_data['entityType']}/{name}",
}
),
content=f"# {name}\n",
observations=[
Observation(content=obs)
for obs in entity_data["observations"]
],
relations=entity_relations.get(name, []) # Add any relations where this entity is the source
observations=[Observation(content=obs) for obs in entity_data["observations"]],
relations=entity_relations.get(
name, []
), # Add any relations where this entity is the source
)
# Let markdown processor handle writing
file_path = base_path / f"{entity_data['entityType']}/{name}.md"
await markdown_processor.write_file(file_path, entity)
entities_created += 1
progress.update(write_task, advance=1)
return {
"entities": entities_created,
"relations": sum(len(rels) for rels in entity_relations.values())
"relations": sum(len(rels) for rels in entity_relations.values()),
}
@@ -101,39 +103,41 @@ def import_json(
json_path: Path = typer.Argument(..., help="Path to memory.json file to import"),
):
"""Import entities and relations from a memory.json file.
This command will:
1. Read entities and relations from the JSON file
2. Create markdown files for each entity
3. Include outgoing relations in each entity's markdown
After importing, run 'basic-memory sync' to index the new files.
"""
if not json_path.exists():
typer.echo(f"Error: File not found: {json_path}", err=True)
raise typer.Exit(1)
try:
# Get markdown processor
markdown_processor = asyncio.run(get_markdown_processor())
# Process the file
base_path = config.home
console.print(f"\nImporting from {json_path}...writing to {base_path}")
results = asyncio.run(process_memory_json(json_path, base_path, markdown_processor))
# Show results
console.print(Panel(
f"[green]Import complete![/green]\n\n"
f"Created {results['entities']} entities\n"
f"Added {results['relations']} relations",
expand=False
))
console.print(
Panel(
f"[green]Import complete![/green]\n\n"
f"Created {results['entities']} entities\n"
f"Added {results['relations']} relations",
expand=False,
)
)
console.print("\nRun 'basic-memory sync' to index the new files.")
except Exception as e:
logger.exception("Import failed")
typer.echo(f"Error during import: {e}", err=True)
raise typer.Exit(1)
raise typer.Exit(1)
+20
View File
@@ -0,0 +1,20 @@
"""MCP server command."""
from loguru import logger
from basic_memory.cli.app import app
from basic_memory.config import config
# Import mcp instance
from basic_memory.mcp.server import mcp as mcp_server # pragma: no cover
# Import mcp tools to register them
import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
@app.command()
def mcp():
"""Run the MCP server for Claude Desktop integration."""
home_dir = config.home
logger.info("Starting Basic Memory MCP server")
logger.info(f"Home directory: {home_dir}")
mcp_server.run()
+10 -6
View File
@@ -21,7 +21,9 @@ from basic_memory.sync.utils import SyncReport
console = Console()
async def get_file_change_scanner(db_type=DatabaseType.FILESYSTEM) -> FileChangeScanner:
async def get_file_change_scanner(
db_type=DatabaseType.FILESYSTEM,
) -> FileChangeScanner: # pragma: no cover
"""Get sync service instance."""
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
engine,
@@ -32,7 +34,9 @@ async def get_file_change_scanner(db_type=DatabaseType.FILESYSTEM) -> FileChange
return file_change_scanner
def add_files_to_tree(tree: Tree, paths: Set[str], style: str, checksums: Dict[str, str] = None):
def add_files_to_tree(
tree: Tree, paths: Set[str], style: str, checksums: Dict[str, str] | None = None
):
"""Add files to tree, grouped by directory."""
# Group by directory
by_dir = {}
@@ -126,7 +130,8 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False):
by_dir = group_changes_by_directory(changes)
for dir_name, counts in sorted(by_dir.items()):
summary = build_directory_summary(counts)
tree.add(f"[bold]{dir_name}/[/bold] {summary}")
if summary: # Only show directories with changes
tree.add(f"[bold]{dir_name}/[/bold] {summary}")
console.print(Panel(tree, expand=False))
@@ -145,8 +150,7 @@ def status(
"""Show sync status between files and database."""
try:
sync_service = asyncio.run(get_file_change_scanner())
asyncio.run(run_status(sync_service, verbose))
asyncio.run(run_status(sync_service, verbose)) # pragma: no cover
except Exception as e:
logger.exception(f"Error checking status: {e}")
typer.echo(f"Error checking status: {e}", err=True)
raise typer.Exit(1)
raise typer.Exit(code=1) # pragma: no cover
+5 -56
View File
@@ -9,9 +9,6 @@ from typing import List, Dict
import typer
from loguru import logger
from rich.console import Console
from rich.padding import Padding
from rich.panel import Panel
from rich.text import Text
from rich.tree import Tree
from basic_memory import db
@@ -42,7 +39,7 @@ class ValidationIssue:
error: str
async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
async def get_sync_service(db_type=DatabaseType.FILESYSTEM): # pragma: no cover
"""Get sync service instance with all dependencies."""
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
engine,
@@ -97,53 +94,6 @@ def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[V
return dict(grouped)
def display_validation_errors(issues: List[ValidationIssue]):
"""Display validation errors in a rich, organized format."""
# Create header
console.print()
console.print(
Panel("[red bold]Error:[/red bold] Invalid frontmatter in knowledge files", expand=False)
)
console.print()
# Group issues by directory
grouped_issues = group_issues_by_directory(issues)
# Create tree structure
tree = Tree("Knowledge Files")
for dir_name, dir_issues in sorted(grouped_issues.items()):
# Create branch for directory
branch = tree.add(
f"[bold blue]{dir_name}/[/bold blue] ([yellow]{len(dir_issues)} files[/yellow])"
)
# Add each file issue
for issue in sorted(dir_issues, key=lambda x: x.file_path):
file_name = Path(issue.file_path).name
branch.add(
Text.assemble(("└─ ", "dim"), (file_name, "yellow"), ": ", (issue.error, "red"))
)
# Display tree
console.print(Padding(tree, (1, 2)))
# Add help text
console.print()
console.print(
Panel(
Text.assemble(
("To fix:", "bold"),
"\n1. Add required frontmatter fields to each file",
"\n2. Run ",
("basic-memory sync", "bold cyan"),
" again",
),
expand=False,
)
)
console.print()
def display_sync_summary(knowledge: SyncReport):
"""Display a one-line summary of sync changes."""
total_changes = knowledge.total_changes
@@ -204,7 +154,6 @@ def display_detailed_sync_results(knowledge: SyncReport):
async def run_sync(verbose: bool = False, watch: bool = False):
"""Run sync operation."""
sync_service = await get_sync_service()
# Start watching if requested
@@ -212,10 +161,10 @@ async def run_sync(verbose: bool = False, watch: bool = False):
watch_service = WatchService(
sync_service=sync_service,
file_service=sync_service.entity_service.file_service,
config=config
config=config,
)
await watch_service.handle_changes(config.home)
await watch_service.run()
await watch_service.run() # pragma: no cover
else:
# one time sync
knowledge_changes = await sync_service.sync(config.home)
@@ -223,7 +172,7 @@ async def run_sync(verbose: bool = False, watch: bool = False):
if verbose:
display_detailed_sync_results(knowledge_changes)
else:
display_sync_summary(knowledge_changes)
display_sync_summary(knowledge_changes) # pragma: no cover
@app.command()
@@ -246,7 +195,7 @@ def sync(
# Run sync
asyncio.run(run_sync(verbose=verbose, watch=watch))
except Exception as e:
except Exception as e: # pragma: no cover
if not isinstance(e, typer.Exit):
logger.exception("Sync failed")
typer.echo(f"Error during sync: {e}", err=True)
+5 -38
View File
@@ -1,47 +1,14 @@
"""Main CLI entry point for basic-memory."""
import sys
"""Main CLI entry point for basic-memory.""" # pragma: no cover
import typer
from loguru import logger
from basic_memory.cli.app import app
from basic_memory.cli.app import app # pragma: no cover
from basic_memory.utils import setup_logging # pragma: no cover
# Register commands
from basic_memory.cli.commands import status, sync
__all__ = ["status", "sync"]
from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp # noqa: F401 # pragma: no cover
from basic_memory.config import config
def setup_logging(home_dir: str = config.home, log_file: str = ".basic-memory/basic-memory-tools.log"):
"""Configure logging for the application."""
# Remove default handler and any existing handlers
logger.remove()
# Add file handler for debug level logs
log = f"{home_dir}/{log_file}"
logger.add(
log,
level="DEBUG",
rotation="100 MB",
retention="10 days",
backtrace=True,
diagnose=True,
enqueue=True,
colorize=False,
)
# Add stderr handler for warnings and errors only
logger.add(
sys.stderr,
level="WARNING",
backtrace=True,
diagnose=True
)
# Set up logging when module is imported
setup_logging()
setup_logging(log_file=".basic-memory/basic-memory-cli.log") # pragma: no cover
if __name__ == "__main__": # pragma: no cover
app()
+3 -3
View File
@@ -2,7 +2,6 @@
from pathlib import Path
from loguru import logger
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -27,6 +26,8 @@ class ProjectConfig(BaseSettings):
default=500, description="Milliseconds to wait after changes before syncing", gt=0
)
log_level: str = "INFO"
model_config = SettingsConfigDict(
env_prefix="BASIC_MEMORY_",
extra="ignore",
@@ -45,7 +46,7 @@ class ProjectConfig(BaseSettings):
@field_validator("home")
@classmethod
def ensure_path_exists(cls, v: Path) -> Path:
def ensure_path_exists(cls, v: Path) -> Path: # pragma: no cover
"""Ensure project path exists."""
if not v.exists():
v.mkdir(parents=True)
@@ -54,4 +55,3 @@ class ProjectConfig(BaseSettings):
# Load project config
config = ProjectConfig()
logger.info(f"project config home: {config.home}")
+15 -22
View File
@@ -14,9 +14,8 @@ from sqlalchemy.ext.asyncio import (
async_scoped_session,
)
from basic_memory.models import Base, SCHEMA_VERSION
from basic_memory.models import Base
from basic_memory.models.search import CREATE_SEARCH_INDEX
from basic_memory.repository.search_repository import SearchRepository
# Module level state
_engine: Optional[AsyncEngine] = None
@@ -72,6 +71,8 @@ async def scoped_session(
async def init_db() -> None:
"""Initialize database with required tables."""
if _session_maker is None: # pragma: no cover
raise RuntimeError("Database session maker not initialized")
logger.info("Initializing database...")
@@ -85,25 +86,11 @@ async def init_db() -> None:
await session.commit()
async def drop_db():
"""Drop all database tables."""
global _engine, _session_maker
logger.info("Dropping tables...")
async with scoped_session(_session_maker) as session:
conn = await session.connection()
await conn.run_sync(Base.metadata.drop_all)
await session.commit()
# reset global engine and session_maker
_engine = None
_session_maker = None
async def get_or_create_db(
db_path: Path,
db_type: DatabaseType = DatabaseType.FILESYSTEM,
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]: # pragma: no cover
"""Get or create database engine and session maker."""
global _engine, _session_maker
@@ -116,10 +103,12 @@ async def get_or_create_db(
# Initialize database
await init_db()
assert _engine is not None # for type checker
assert _session_maker is not None # for type checker
return _engine, _session_maker
async def shutdown_db():
async def shutdown_db() -> None: # pragma: no cover
"""Clean up database connections."""
global _engine, _session_maker
@@ -129,7 +118,6 @@ async def shutdown_db():
_session_maker = None
@asynccontextmanager
async def engine_session_factory(
db_path: Path,
@@ -143,10 +131,10 @@ async def engine_session_factory(
"""
global _engine, _session_maker
db_url = DatabaseType.get_db_url(db_path, db_type)
logger.debug(f"Creating engine for db_url: {db_url}")
_engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
try:
_session_maker = async_sessionmaker(_engine, expire_on_commit=False)
@@ -154,6 +142,11 @@ async def engine_session_factory(
if init:
await init_db()
assert _engine is not None # for type checker
assert _session_maker is not None # for type checker
yield _engine, _session_maker
finally:
await _engine.dispose()
if _engine:
await _engine.dispose()
_engine = None
_session_maker = None
+3 -4
View File
@@ -29,11 +29,11 @@ from basic_memory.services.search_service import SearchService
## project
def get_project_config() -> ProjectConfig:
def get_project_config() -> ProjectConfig: # pragma: no cover
return config
ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]
ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)] # pragma: no cover
## sqlalchemy
@@ -41,7 +41,7 @@ ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]
async def get_engine_factory(
project_config: ProjectConfigDep,
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]: # pragma: no cover
"""Get engine and session maker."""
return await db.get_or_create_db(project_config.database_path)
@@ -129,7 +129,6 @@ async def get_file_service(
FileServiceDep = Annotated[FileService, Depends(get_file_service)]
async def get_entity_service(
entity_repository: EntityRepositoryDep,
observation_repository: ObservationRepositoryDep,
+36 -35
View File
@@ -1,7 +1,8 @@
"""Utilities for file operations."""
import hashlib
from pathlib import Path
from typing import Dict, Any, Tuple
from typing import Dict, Any
import yaml
from loguru import logger
@@ -9,35 +10,38 @@ from loguru import logger
class FileError(Exception):
"""Base exception for file operations."""
pass
class FileWriteError(FileError):
"""Raised when file operations fail."""
pass
class ParseError(FileError):
"""Raised when parsing file content fails."""
pass
async def compute_checksum(content: str) -> str:
"""
Compute SHA-256 checksum of content.
Args:
content: Text content to hash
Returns:
SHA-256 hex digest
Raises:
FileError: If checksum computation fails
"""
try:
return hashlib.sha256(content.encode()).hexdigest()
except Exception as e:
except Exception as e: # pragma: no cover
logger.error(f"Failed to compute checksum: {e}")
raise FileError(f"Failed to compute checksum: {e}")
@@ -45,16 +49,16 @@ async def compute_checksum(content: str) -> str:
async def ensure_directory(path: Path) -> None:
"""
Ensure directory exists, creating if necessary.
Args:
path: Directory path to ensure
Raises:
FileWriteError: If directory creation fails
"""
try:
path.mkdir(parents=True, exist_ok=True)
except Exception as e:
except Exception as e: # pragma: no cover
logger.error(f"Failed to create directory: {path}: {e}")
raise FileWriteError(f"Failed to create directory {path}: {e}")
@@ -62,22 +66,20 @@ async def ensure_directory(path: Path) -> None:
async def write_file_atomic(path: Path, content: str) -> None:
"""
Write file with atomic operation using temporary file.
Args:
path: Target file path
content: Content to write
Raises:
FileWriteError: If write operation fails
"""
temp_path = path.with_suffix(".tmp")
try:
temp_path.write_text(content)
# TODO check for path.exists()
temp_path.replace(path)
logger.debug(f"wrote file: {path}")
except Exception as e:
except Exception as e: # pragma: no cover
temp_path.unlink(missing_ok=True)
logger.error(f"Failed to write file: {path}: {e}")
raise FileWriteError(f"Failed to write file {path}: {e}")
@@ -85,16 +87,19 @@ async def write_file_atomic(path: Path, content: str) -> None:
def has_frontmatter(content: str) -> bool:
"""
Check if content contains YAML frontmatter.
Check if content contains valid YAML frontmatter.
Args:
content: Content to check
Returns:
True if content has frontmatter delimiter (---), False otherwise
True if content has valid frontmatter markers (---), False otherwise
"""
content = content.strip()
return content.startswith("---") and "---" in content[3:]
if not content.startswith("---"):
return False
return "---" in content[3:]
def parse_frontmatter(content: str) -> Dict[str, Any]:
@@ -111,7 +116,7 @@ def parse_frontmatter(content: str) -> Dict[str, Any]:
ParseError: If frontmatter is invalid or parsing fails
"""
try:
if not has_frontmatter(content):
if not content.strip().startswith("---"):
raise ParseError("Content has no frontmatter")
# Split on first two occurrences of ---
@@ -132,7 +137,7 @@ def parse_frontmatter(content: str) -> Dict[str, Any]:
except yaml.YAMLError as e:
raise ParseError(f"Invalid YAML in frontmatter: {e}")
except Exception as e:
except Exception as e: # pragma: no cover
if not isinstance(e, ParseError):
logger.error(f"Failed to parse frontmatter: {e}")
raise ParseError(f"Failed to parse frontmatter: {e}")
@@ -147,27 +152,23 @@ def remove_frontmatter(content: str) -> str:
content: Content with frontmatter
Returns:
Content with frontmatter removed
Content with frontmatter removed, or original content if no frontmatter
Raises:
ParseError: If frontmatter format is invalid
ParseError: If content starts with frontmatter marker but is malformed
"""
try:
if not has_frontmatter(content):
return content.strip()
content = content.strip()
# Split on first two occurrences of ---
parts = content.split("---", 2)
if len(parts) < 3:
raise ParseError("Invalid frontmatter format")
# Return as-is if no frontmatter marker
if not content.startswith("---"):
return content
return parts[2].strip()
# Split on first two occurrences of ---
parts = content.split("---", 2)
if len(parts) < 3:
raise ParseError("Invalid frontmatter format")
except Exception as e:
if not isinstance(e, ParseError):
logger.error(f"Failed to remove frontmatter: {e}")
raise ParseError(f"Failed to remove frontmatter: {e}")
raise
return parts[2].strip()
async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
@@ -208,6 +209,6 @@ async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
await write_file_atomic(path, final_content)
return await compute_checksum(final_content)
except Exception as e:
except Exception as e: # pragma: no cover
logger.error(f"Failed to update frontmatter in {path}: {e}")
raise FileError(f"Failed to update frontmatter: {e}")
raise FileError(f"Failed to update frontmatter: {e}")
+13 -30
View File
@@ -2,6 +2,7 @@
Uses markdown-it with plugins to parse structured data from markdown content.
"""
from dataclasses import dataclass, field
from pathlib import Path
from datetime import datetime
@@ -21,11 +22,12 @@ from basic_memory.markdown.schemas import (
md = MarkdownIt().use(observation_plugin).use(relation_plugin)
@dataclass
class EntityContent:
content: str
observations: list[Observation] = field(default_factory=list)
relations: list[Relation] = field(default_factory=list)
content: str
observations: list[Observation] = field(default_factory=list)
relations: list[Relation] = field(default_factory=list)
def parse(content: str) -> EntityContent:
@@ -53,13 +55,13 @@ def parse(content: str) -> EntityContent:
relations=relations,
)
def parse_tags(tags: Any) -> list[str]:
"""Parse tags into list of strings."""
if isinstance(tags, str):
return [t.strip() for t in tags.split(",") if t.strip()]
if isinstance(tags, (list, tuple)):
return [str(t).strip() for t in tags if str(t).strip()]
return []
return [t.strip() for t in tags.split(",") if t.strip()]
class EntityParser:
"""Parser for markdown files into Entity objects."""
@@ -68,21 +70,6 @@ class EntityParser:
"""Initialize parser with base path for relative permalink generation."""
self.base_path = base_path.resolve()
def relative_path(self, file_path: Path) -> str:
"""Get file path relative to base_path.
Example:
base_path: /project/root
file_path: /project/root/design/models/data.md
returns: "design/models/data"
"""
# Get relative path and remove .md extension
rel_path = file_path.resolve().relative_to(self.base_path)
if rel_path.suffix.lower() == ".md":
return str(rel_path.with_suffix(""))
return str(rel_path)
def parse_date(self, value: Any) -> Optional[datetime]:
"""Parse date strings using dateparser for maximum flexibility.
@@ -96,21 +83,18 @@ class EntityParser:
if isinstance(value, datetime):
return value
if isinstance(value, str):
try:
parsed = dateparser.parse(value)
if parsed:
return parsed
except Exception:
pass
parsed = dateparser.parse(value)
if parsed:
return parsed
return None
async def parse_file(self, file_path: Path) -> EntityMarkdown:
"""Parse markdown file into EntityMarkdown."""
absolute_path = self.base_path / file_path
# Parse frontmatter and content using python-frontmatter
post = frontmatter.load(str(absolute_path))
# Extract file stat info
file_stats = absolute_path.stat()
@@ -134,4 +118,3 @@ class EntityParser:
created=datetime.fromtimestamp(file_stats.st_ctime),
modified=datetime.fromtimestamp(file_stats.st_mtime),
)
@@ -19,7 +19,7 @@ class DirtyFileError(Exception):
class MarkdownProcessor:
"""Process markdown files while preserving content and structure.
used only for import
This class handles the file I/O aspects of our markdown processing. It:
@@ -90,14 +90,14 @@ class MarkdownProcessor:
# Convert frontmatter to dict
frontmatter_dict = OrderedDict()
frontmatter_dict["title"] = markdown.frontmatter.title
frontmatter_dict["type"] = markdown.frontmatter.type
frontmatter_dict["title"] = markdown.frontmatter.title
frontmatter_dict["type"] = markdown.frontmatter.type
frontmatter_dict["permalink"] = markdown.frontmatter.permalink
metadata = markdown.frontmatter.metadata or {}
for k,v in metadata.items():
for k, v in metadata.items():
frontmatter_dict[k] = v
# Start with user content (or minimal title for new files)
content = markdown.content or f"# {markdown.frontmatter.title}\n"
@@ -107,7 +107,7 @@ class MarkdownProcessor:
# add a blank line if we have semantic content
if markdown.observations or markdown.relations:
content += "\n"
if markdown.observations:
content += self.format_observations(markdown.observations)
if markdown.relations:
+109 -123
View File
@@ -8,19 +8,19 @@ from markdown_it.token import Token
# Observation handling functions
def is_observation(token: Token) -> bool:
"""Check if token looks like our observation format."""
if token.type != 'inline':
if token.type != "inline": # pragma: no cover
return False
content = token.content.strip()
if not content:
if not content: # pragma: no cover
return False
# if it's a markdown_task, return false
if content.startswith('[ ]') or content.startswith('[x]') or content.startswith('[-]'):
if content.startswith("[ ]") or content.startswith("[x]") or content.startswith("[-]"):
return False
has_category = content.startswith('[') and ']' in content
has_tags = '#' in content
has_category = content.startswith("[") and "]" in content
has_tags = "#" in content
return has_category or has_tags
@@ -28,119 +28,126 @@ def parse_observation(token: Token) -> Dict[str, Any]:
"""Extract observation parts from token."""
# Strip bullet point if present
content = token.content.strip()
if content.startswith('- '):
content = content[2:].strip()
elif content.startswith('-'):
content = content[1:].strip()
# Parse [category]
category = None
if content.startswith('['):
end = content.find(']')
if content.startswith("["):
end = content.find("]")
if end != -1:
category = content[1:end].strip() or None # Convert empty to None
content = content[end + 1:].strip()
content = content[end + 1 :].strip()
# Parse (context)
context = None
if content.endswith(')'):
start = content.rfind('(')
if content.endswith(")"):
start = content.rfind("(")
if start != -1:
context = content[start + 1:-1].strip()
context = content[start + 1 : -1].strip()
content = content[:start].strip()
# Parse #tags and content
# Extract tags and keep original content
tags = []
parts = content.split()
content_parts = []
tags = set() # Use set to avoid duplicates
for part in parts:
if part.startswith('#'):
if part.startswith("#"):
# Handle multiple #tags stuck together
if '#' in part[1:]:
if "#" in part[1:]:
# Split on # but keep non-empty tags
subtags = [t for t in part.split('#') if t]
tags.update(subtags)
subtags = [t for t in part.split("#") if t]
tags.extend(subtags)
else:
tags.add(part[1:])
else:
content_parts.append(part)
tags.append(part[1:])
return {
'category': category,
'content': content,
'tags': list(tags) if tags else None,
'context': context
"category": category,
"content": content,
"tags": tags if tags else None,
"context": context,
}
# Relation handling functions
def is_explicit_relation(token: Token) -> bool:
"""Check if token looks like our relation format."""
if token.type != 'inline':
if token.type != "inline": # pragma: no cover
return False
content = token.content.strip()
return '[[' in content and ']]' in content
return "[[" in content and "]]" in content
def parse_relation(token: Token) -> Dict[str, Any]:
def parse_relation(token: Token) -> Dict[str, Any] | None:
"""Extract relation parts from token."""
# Remove bullet point if present
content = token.content.strip()
if content.startswith('- '):
content = content[2:].strip()
elif content.startswith('-'):
content = content[1:].strip()
# Extract [[target]]
target = None
rel_type = 'relates_to' # default
rel_type = "relates_to" # default
context = None
start = content.find('[[')
end = content.find(']]')
start = content.find("[[")
end = content.find("]]")
if start != -1 and end != -1:
# Get text before link as relation type
before = content[:start].strip()
if before:
rel_type = before
# Get target
target = content[start + 2:end].strip()
target = content[start + 2 : end].strip()
# Look for context after
after = content[end + 2:].strip()
if after.startswith('(') and after.endswith(')'):
after = content[end + 2 :].strip()
if after.startswith("(") and after.endswith(")"):
context = after[1:-1].strip() or None
if not target:
if not target: # pragma: no cover
return None
return {
'type': rel_type,
'target': target,
'context': context
}
return {"type": rel_type, "target": target, "context": context}
def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
"""Find wiki-style links in regular content."""
relations = []
import re
pattern = r'\[\[([^\]]+)\]\]'
for match in re.finditer(pattern, content):
target = match.group(1).strip()
if target and not target.startswith('[['): # Avoid nested matches
relations.append({
'type': 'links to',
'target': target,
'context': None
})
start = 0
while True:
# Find next outer-most [[
start = content.find("[[", start)
if start == -1: # pragma: no cover
break
# Find matching ]]
depth = 1
pos = start + 2
end = -1
while pos < len(content):
if content[pos : pos + 2] == "[[":
depth += 1
pos += 2
elif content[pos : pos + 2] == "]]":
depth -= 1
if depth == 0:
end = pos
break
pos += 2
else:
pos += 1
if end == -1:
# No matching ]] found
break
target = content[start + 2 : end].strip()
if target:
relations.append({"type": "links to", "target": target, "context": None})
start = end + 2
return relations
@@ -149,88 +156,67 @@ def observation_plugin(md: MarkdownIt) -> None:
- [category] Content text #tag1 #tag2 (context)
- Content text #tag1 (context) # No category is also valid
"""
def observation_rule(state: Any) -> None:
"""Process observations in token stream."""
tokens = state.tokens
current_section = None
in_list_item = False
for idx in range(len(tokens)):
token = tokens[idx]
# Track current section by headings
if token.type == 'heading_open':
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
if next_token and next_token.type == 'inline':
current_section = next_token.content.lower()
# Track list nesting
elif token.type == 'list_item_open':
in_list_item = True
elif token.type == 'list_item_close':
in_list_item = False
# Initialize meta for all tokens
token.meta = token.meta or {}
# Parse observations in list items
if token.type == 'inline' and is_observation(token):
if token.type == "inline" and is_observation(token):
obs = parse_observation(token)
if obs['content']: # Only store if we have content
token.meta['observation'] = obs
if obs["content"]: # Only store if we have content
token.meta["observation"] = obs
# Add the rule after inline processing
md.core.ruler.after('inline', 'observations', observation_rule)
md.core.ruler.after("inline", "observations", observation_rule)
def relation_plugin(md: MarkdownIt) -> None:
"""Plugin for parsing relation formats:
Explicit relations:
- relation_type [[target]] (context)
Implicit relations (links in content):
Some text with [[target]] reference
"""
def relation_rule(state: Any) -> None:
"""Process relations in token stream."""
tokens = state.tokens
current_section = None
in_list_item = False
for idx in range(len(tokens)):
token = tokens[idx]
# Track current section by headings
if token.type == 'heading_open':
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
if next_token and next_token.type == 'inline':
current_section = next_token.content.lower()
# Track list nesting
elif token.type == 'list_item_open':
if token.type == "list_item_open":
in_list_item = True
elif token.type == 'list_item_close':
elif token.type == "list_item_close":
in_list_item = False
# Initialize meta for all tokens
token.meta = token.meta or {}
# Only process inline tokens
if token.type == 'inline':
if token.type == "inline":
# Check for explicit relations in list items
if in_list_item and is_explicit_relation(token):
rel = parse_relation(token)
if rel:
token.meta['relations'] = [rel]
token.meta["relations"] = [rel]
# Always check for inline links in any text
elif '[[' in token.content:
elif "[[" in token.content:
rels = parse_inline_relations(token.content)
if rels:
token.meta['relations'] = token.meta.get('relations', []) + rels
token.meta["relations"] = token.meta.get("relations", []) + rels
# Add the rule after inline processing
md.core.ruler.after('inline', 'relations', relation_rule)
md.core.ruler.after("inline", "relations", relation_rule)
+7 -8
View File
@@ -13,7 +13,7 @@ class Observation(BaseModel):
content: str
tags: Optional[List[str]] = None
context: Optional[str] = None
def __str__(self) -> str:
obs_string = f"- [{self.category}] {self.content}"
if self.context:
@@ -27,7 +27,7 @@ class Relation(BaseModel):
type: str
target: str
context: Optional[str] = None
def __str__(self) -> str:
rel_string = f"- {self.type} [[{self.target}]]"
if self.context:
@@ -38,24 +38,23 @@ class Relation(BaseModel):
class EntityFrontmatter(BaseModel):
"""Required frontmatter fields for an entity."""
metadata: Optional[dict] = None
metadata: dict = {}
@property
def tags(self) -> List[str]:
return self.metadata.get("tags") if self.metadata else []
return self.metadata.get("tags") if self.metadata else [] # pyright: ignore
@property
def title(self) -> str:
return self.metadata.get("title") if self.metadata else None
return self.metadata.get("title") if self.metadata else None # pyright: ignore
@property
def type(self) -> str:
return self.metadata.get("type", "note") if self.metadata else "note"
return self.metadata.get("type", "note") if self.metadata else "note" # pyright: ignore
@property
def permalink(self) -> str:
return self.metadata.get("permalink") if self.metadata else None
return self.metadata.get("permalink") if self.metadata else None # pyright: ignore
class EntityMarkdown(BaseModel):
+71 -122
View File
@@ -1,144 +1,93 @@
"""Utilities for converting between markdown and entity models."""
from pathlib import Path
from typing import Optional
from typing import Optional, Any
from frontmatter import Post
from basic_memory.markdown import EntityMarkdown, EntityFrontmatter, Observation, Relation
from basic_memory.markdown.entity_parser import parse
from basic_memory.models import Entity, ObservationCategory, Observation as ObservationModel
from basic_memory.markdown import EntityMarkdown
from basic_memory.models import Entity, Observation as ObservationModel
from basic_memory.utils import generate_permalink
def entity_model_to_markdown(entity: Entity, content: Optional[str] = None) -> EntityMarkdown:
def entity_model_from_markdown(
file_path: Path, markdown: EntityMarkdown, entity: Optional[Entity] = None
) -> Entity:
"""
Converts an entity model to its Markdown representation, including metadata,
observations, relations, and content. Ensures that observations and relations
from the provided content are synchronized with the entity model. Removes
duplicate or unmatched observations and relations from the content to maintain
consistency.
:param entity: An instance of the Entity class containing metadata, observations,
relations, and other properties of the entity.
:type entity: Entity
:param content: Optional raw Markdown-formatted content to be parsed for semantic
information like observations or relations.
:type content: Optional[str]
:return: An instance of the EntityMarkdown class containing the entity's
frontmatter, observations, relations, and sanitized content formatted
in Markdown.
:rtype: EntityMarkdown
"""
metadata = entity.entity_metadata or {}
metadata["type"] = entity.entity_type or "note"
metadata["title"] = entity.title
metadata["permalink"] = entity.permalink
# convert model to markdown
entity_observations = [
Observation(
category=obs.category,
content=obs.content,
tags=obs.tags if obs.tags else None,
context=obs.context,
)
for obs in entity.observations
]
entity_relations = [
Relation(
type=r.relation_type,
target=r.to_entity.title if r.to_entity else r.to_name,
context=r.context,
)
for r in entity.outgoing_relations
]
observations = entity_observations
relations = entity_relations
# parse the content to see if it has semantic info (observations/relations)
entity_content = parse(content) if content else None
if entity_content:
# remove if they are already in the content
observations = [o for o in entity_observations if o not in entity_content.observations]
relations = [r for r in entity_relations if r not in entity_content.relations]
# remove from the content if not present in the db entity
for o in entity_content.observations:
if o not in entity_observations:
content = content.replace(str(o), "")
for r in entity_content.relations:
if r not in entity_relations:
content = content.replace(str(r), "")
return EntityMarkdown(
frontmatter=EntityFrontmatter(metadata=metadata),
content=content,
observations=observations,
relations=relations,
created = entity.created_at,
modified = entity.updated_at,
)
def entity_model_from_markdown(file_path: Path, markdown: EntityMarkdown, entity: Optional[Entity] = None) -> Entity:
"""
Convert markdown entity to model.
Does not include relations.
Convert markdown entity to model. Does not include relations.
Args:
file_path: Path to the markdown file
markdown: Parsed markdown entity
include_relations: Whether to include relations. Set False for first sync pass.
entity: Optional existing entity to update
Returns:
Entity model populated from markdown
Raises:
ValueError: If required datetime fields are missing from markdown
"""
# Validate/default category
def get_valid_category(obs):
if not obs.category or obs.category not in [c.value for c in ObservationCategory]:
return ObservationCategory.NOTE.value
return obs.category
if not markdown.created or not markdown.modified: # pragma: no cover
raise ValueError("Both created and modified dates are required in markdown")
# Generate permalink if not provided
permalink = markdown.frontmatter.permalink or generate_permalink(file_path)
# Create or update entity
model = entity or Entity()
model.title=markdown.frontmatter.title
model.entity_type=markdown.frontmatter.type
model.permalink=permalink
model.file_path=str(file_path)
model.content_type="text/markdown"
model.created_at=markdown.created
model.updated_at=markdown.modified
model.entity_metadata={k:str(v) for k,v in markdown.frontmatter.metadata.items()}
model.observations=[
ObservationModel(
content=obs.content,
category=get_valid_category(obs),
context=obs.context,
tags=obs.tags,
)
for obs in markdown.observations
]
# Update basic fields
model.title = markdown.frontmatter.title
model.entity_type = markdown.frontmatter.type
model.permalink = permalink
model.file_path = str(file_path)
model.content_type = "text/markdown"
model.created_at = markdown.created
model.updated_at = markdown.modified
# Handle metadata - ensure all values are strings and filter None
metadata = markdown.frontmatter.metadata or {}
model.entity_metadata = {k: str(v) for k, v in metadata.items() if v is not None}
# Convert observations
model.observations = [
ObservationModel(
content=obs.content,
category=obs.category,
context=obs.context,
tags=obs.tags,
)
for obs in markdown.observations
]
return model
async def schema_to_markdown(schema):
async def schema_to_markdown(schema: Any) -> Post:
"""
Convert schema to markdown.
:param schema: the schema to convert
:return: Post
Convert schema to markdown Post object.
Args:
schema: Schema to convert (must have title, entity_type, and permalink attributes)
Returns:
Post object with frontmatter metadata
"""
# Create Post object
# Extract content and metadata
content = schema.content or ""
frontmatter_metadata = schema.entity_metadata or {}
# remove from map so we can define ordering in frontmatter
if "type" in frontmatter_metadata:
del frontmatter_metadata["type"]
if "title" in frontmatter_metadata:
del frontmatter_metadata["title"]
if "permalink" in frontmatter_metadata:
del frontmatter_metadata["permalink"]
post = Post(content, title=schema.title, type=schema.entity_type, permalink=schema.permalink, **frontmatter_metadata)
frontmatter_metadata = dict(schema.entity_metadata or {})
# Remove special fields for ordered frontmatter
for field in ["type", "title", "permalink"]:
frontmatter_metadata.pop(field, None)
# Create Post with ordered fields
post = Post(
content,
title=schema.title,
type=schema.entity_type,
permalink=schema.permalink,
**frontmatter_metadata,
)
return post
+1 -1
View File
@@ -1 +1 @@
"""MCP server for basic-memory."""
"""MCP server for basic-memory."""
-2
View File
@@ -6,5 +6,3 @@ BASE_URL = "http://test"
# Create shared async client
client = AsyncClient(transport=ASGITransport(app=fastapi_app), base_url=BASE_URL)
-21
View File
@@ -1,21 +0,0 @@
"""Main MCP entrypoint for Basic Memory.
Creates and configures the shared MCP instance and handles server startup.
"""
from loguru import logger
from basic_memory.config import config
# Import shared mcp instance
from basic_memory.mcp.server import mcp
# Import tools to register them
import basic_memory.mcp.tools # noqa: F401
if __name__ == "__main__":
home_dir = config.home
logger.info("Starting Basic Memory MCP server")
logger.info(f"Home directory: {home_dir}")
mcp.run()
+3 -27
View File
@@ -1,39 +1,15 @@
"""Enhanced FastMCP server instance for Basic Memory."""
import sys
from loguru import logger
from mcp.server.fastmcp import FastMCP
from mcp.server.fastmcp.utilities.logging import configure_logging
from basic_memory.config import config
from basic_memory.utils import setup_logging
# mcp console logging
configure_logging(level="INFO")
# configure_logging(level='INFO')
def setup_logging(home_dir: str = config.home, log_file: str = ".basic-memory/basic-memory.log"):
"""Configure file logging to the basic-memory home directory."""
log = f"{home_dir}/{log_file}"
# Add file handler with rotation
logger.add(
log,
rotation="100 MB",
retention="10 days",
backtrace=True,
diagnose=True,
enqueue=True,
colorize=False,
)
# Add stderr handler
logger.add(
sys.stderr,
colorize=True,
)
# start our out file logging
setup_logging()
setup_logging(log_file=".basic-memory/basic-memory.log")
# Create the shared server instance
mcp = FastMCP("Basic Memory")
+5 -3
View File
@@ -7,8 +7,10 @@ all tools with the MCP server.
# Import tools to register them with MCP
from basic_memory.mcp.tools.memory import build_context, recent_activity
#from basic_memory.mcp.tools.ai_edit import ai_edit
# from basic_memory.mcp.tools.ai_edit import ai_edit
from basic_memory.mcp.tools.notes import read_note, write_note
from basic_memory.mcp.tools.search import search
from basic_memory.mcp.tools.knowledge import (
delete_entities,
@@ -26,9 +28,9 @@ __all__ = [
# memory tools
"build_context",
"recent_activity",
#notes
# notes
"read_note",
"write_note",
# file edit
#"ai_edit",
# "ai_edit",
]
-84
View File
@@ -1,84 +0,0 @@
"""Tool for AI-assisted file editing."""
from pathlib import Path
from typing import List, Dict, Any
from basic_memory.mcp.server import mcp
def _detect_indent(text: str, match_pos: int) -> int:
"""Get indentation level at a position in text."""
# Find start of line containing the match
line_start = text.rfind("\n", 0, match_pos)
if line_start < 0:
line_start = 0
else:
line_start += 1 # Skip newline char
# Count leading spaces
pos = line_start
while pos < len(text) and text[pos].isspace():
pos += 1
return pos - line_start
def _apply_indent(text: str, spaces: int) -> str:
"""Apply indentation to text."""
prefix = " " * spaces
return "\n".join(prefix + line if line.strip() else line for line in text.split("\n"))
@mcp.tool()
async def ai_edit(path: str, edits: List[Dict[str, Any]]) -> bool:
"""AI-assisted file editing tool.
Args:
path: Path to file to edit
edits: List of edits to apply. Each edit is a dict with:
oldText: Text to replace
newText: New content
options: Optional dict with:
indent: Number of spaces to indent
preserveIndentation: Keep existing indent (default: true)
Returns:
bool: True if edits were applied successfully
"""
try:
# Read file
content = Path(path).read_text()
original = content
success = True
# Apply each edit
for edit in edits:
old_text = edit["oldText"]
new_text = edit["newText"]
options = edit.get("options", {})
# Find text to replace
match_pos = content.find(old_text)
if match_pos < 0:
success = False
continue
# Handle indentation
if not options.get("preserveIndentation", True):
# Use existing indentation
indent = _detect_indent(content, match_pos)
new_text = _apply_indent(new_text, indent)
elif "indent" in options:
# Use specified indentation
new_text = _apply_indent(new_text, options["indent"])
# Apply the edit
content = content.replace(old_text, new_text)
# Write back if changed
if content != original:
Path(path).write_text(content)
return success
except Exception as e:
print(f"Error applying edits: {e}")
return False
+2 -2
View File
@@ -2,7 +2,7 @@
from basic_memory.mcp.server import mcp
from basic_memory.mcp.tools.utils import call_get, call_post
from basic_memory.schemas.base import PathId
from basic_memory.schemas.base import Permalink
from basic_memory.schemas.request import (
GetEntitiesRequest,
)
@@ -16,7 +16,7 @@ from basic_memory.mcp.async_client import client
@mcp.tool(
description="Get complete information about a specific entity including observations and relations",
)
async def get_entity(permalink: PathId) -> EntityResponse:
async def get_entity(permalink: Permalink) -> EntityResponse:
"""Get a specific entity info by its permalink.
Args:
+8 -4
View File
@@ -7,9 +7,13 @@ from loguru import logger
from basic_memory.mcp.async_client import client
from basic_memory.mcp.server import mcp
from basic_memory.mcp.tools.utils import call_get
from basic_memory.schemas.memory import GraphContext, MemoryUrl, memory_url, memory_url_path, normalize_memory_url
from basic_memory.schemas.memory import (
GraphContext,
MemoryUrl,
memory_url_path,
normalize_memory_url,
)
from basic_memory.schemas.base import TimeFrame
from basic_memory.schemas.search import SearchItemType
@mcp.tool(
@@ -84,7 +88,7 @@ async def build_context(
""",
)
async def recent_activity(
type: List[Literal["entity", "observation", "relation"]] = None,
type: List[Literal["entity", "observation", "relation"]] = [],
depth: Optional[int] = 1,
timeframe: Optional[TimeFrame] = "7d",
max_results: int = 10,
@@ -136,7 +140,7 @@ async def recent_activity(
"timeframe": timeframe,
"max_results": max_results,
}
if type:
if type:
params["type"] = type
response = await call_get(
+2 -1
View File
@@ -1,4 +1,5 @@
"""Search tools for Basic Memory MCP server."""
from loguru import logger
from basic_memory.mcp.server import mcp
@@ -24,5 +25,5 @@ async def search(query: SearchQuery) -> SearchResponse:
SearchResponse with search results and metadata
"""
logger.info(f"Searching for {query.text}")
response = await call_post(client,"/search/", json=query.model_dump())
response = await call_post(client, "/search/", json=query.model_dump())
return SearchResponse.model_validate(response.json())
+1 -1
View File
@@ -45,7 +45,7 @@ async def call_get(
return response
except HTTPStatusError as e:
logger.error(f"Error calling GET {url}: {e}")
raise ToolError(f"Error calling tool: {e}. Response: {response.text}") from e
raise ToolError(f"Error calling tool: {e}.") from e
async def call_put(
+1 -2
View File
@@ -2,7 +2,7 @@
import basic_memory
from basic_memory.models.base import Base
from basic_memory.models.knowledge import Entity, Observation, Relation, ObservationCategory
from basic_memory.models.knowledge import Entity, Observation, Relation
SCHEMA_VERSION = basic_memory.__version__ + "-" + "003"
@@ -10,6 +10,5 @@ __all__ = [
"Base",
"Entity",
"Observation",
"ObservationCategory",
"Relation",
]
+3 -3
View File
@@ -1,10 +1,10 @@
"""Base model class for SQLAlchemy models."""
from sqlalchemy import String, Integer
from sqlalchemy.ext.asyncio import AsyncAttrs
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from sqlalchemy.orm import DeclarativeBase
class Base(AsyncAttrs, DeclarativeBase):
"""Base class for all models"""
pass
pass
+23 -60
View File
@@ -1,6 +1,5 @@
"""Knowledge graph models."""
import re
from datetime import datetime
from typing import Optional
@@ -14,17 +13,15 @@ from sqlalchemy import (
Index,
JSON,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
from sqlalchemy.orm import Mapped, mapped_column, relationship
from basic_memory.models.base import Base
from enum import Enum
from basic_memory.utils import generate_permalink
class Entity(Base):
"""
Core entity in the knowledge graph.
"""Core entity in the knowledge graph.
Entities represent semantic nodes maintained by the AI layer. Each entity:
- Has a unique numeric ID (database-generated)
@@ -79,44 +76,15 @@ class Entity(Base):
@property
def relations(self):
"""Get all relations (incoming and outgoing) for this entity."""
return self.incoming_relations + self.outgoing_relations
@validates("permalink")
def validate_permalink(self, key, value):
"""Validate permalink format.
Requirements:
1. Must be valid URI path component
2. Only lowercase letters, numbers, and hyphens (no underscores)
3. Path segments separated by forward slashes
4. No leading/trailing hyphens in segments
"""
if not value:
raise ValueError("Permalink must not be None")
if not re.match(r"^[a-z0-9][a-z0-9\-/]*[a-z0-9]$", value):
raise ValueError(
f"Invalid permalink format: {value}. "
"Use only lowercase letters, numbers, and hyphens."
)
return value
def __repr__(self) -> str:
return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
class ObservationCategory(str, Enum):
TECH = "tech"
DESIGN = "design"
FEATURE = "feature"
NOTE = "note"
ISSUE = "issue"
TODO = "todo"
class Observation(Base):
"""
An observation about an entity.
"""An observation about an entity.
Observations are atomic facts or notes about an entity.
"""
@@ -130,13 +98,8 @@ class Observation(Base):
id: Mapped[int] = mapped_column(Integer, primary_key=True)
entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
content: Mapped[str] = mapped_column(Text)
category: Mapped[str] = mapped_column(
String,
nullable=False,
default=ObservationCategory.NOTE.value,
server_default=ObservationCategory.NOTE.value,
)
context: Mapped[str] = mapped_column(Text, nullable=True)
category: Mapped[str] = mapped_column(String, nullable=False, default="note")
context: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
tags: Mapped[Optional[list[str]]] = mapped_column(
JSON, nullable=True, default=list, server_default="[]"
)
@@ -146,23 +109,21 @@ class Observation(Base):
@property
def permalink(self) -> str:
"""
Create synthetic permalink for the observation
We can construct these because observations are always
defined in and owned by a single entity
"""Create synthetic permalink for the observation.
We can construct these because observations are always defined in
and owned by a single entity.
"""
return generate_permalink(
f"{self.entity.permalink}/observations/{self.category}/{self.content}"
)
def __repr__(self) -> str:
def __repr__(self) -> str: # pragma: no cover
return f"Observation(id={self.id}, entity_id={self.entity_id}, content='{self.content}')"
class Relation(Base):
"""
A directed relation between two entities.
"""
"""A directed relation between two entities."""
__tablename__ = "relation"
__table_args__ = (
@@ -174,12 +135,12 @@ class Relation(Base):
id: Mapped[int] = mapped_column(Integer, primary_key=True)
from_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
to_id: Mapped[int] = mapped_column(
to_id: Mapped[Optional[int]] = mapped_column(
Integer, ForeignKey("entity.id", ondelete="CASCADE"), nullable=True
)
to_name: Mapped[str] = mapped_column(String)
relation_type: Mapped[str] = mapped_column(String)
context: Mapped[str] = mapped_column(Text, nullable=True)
context: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
# Relationships
from_entity = relationship(
@@ -189,15 +150,17 @@ class Relation(Base):
@property
def permalink(self) -> str:
"""Create relation permalink showing the semantic connection:
source/relation_type/target
e.g., "specs/search/implements/features/search-ui"
"""
"""Create relation permalink showing the semantic connection.
Format: source/relation_type/target
Example: "specs/search/implements/features/search-ui"
"""
if self.to_entity:
return generate_permalink(
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
)
return generate_permalink(
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
if self.to_entity
else f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
)
def __repr__(self) -> str:
+1 -1
View File
@@ -31,4 +31,4 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
tokenize='unicode61 tokenchars 0x2F', -- Hex code for /
prefix='1,2,3,4' -- Support longer prefixes for paths
);
""")
""")
+5 -3
View File
@@ -2,6 +2,8 @@ from .entity_repository import EntityRepository
from .observation_repository import ObservationRepository
from .relation_repository import RelationRepository
__all__ = ["EntityRepository", "ObservationRepository", "RelationRepository", ]
__all__ = [
"EntityRepository",
"ObservationRepository",
"RelationRepository",
]
@@ -1,8 +1,8 @@
"""Repository for managing entities in the knowledge graph."""
from typing import List, Optional, Sequence
from pathlib import Path
from typing import List, Optional, Sequence, Union
from sqlalchemy import select, or_, asc
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from sqlalchemy.orm import selectinload
from sqlalchemy.orm.interfaces import LoaderOption
@@ -12,109 +12,57 @@ from basic_memory.repository.repository import Repository
class EntityRepository(Repository[Entity]):
"""Repository for Entity model."""
"""Repository for Entity model.
Note: All file paths are stored as strings in the database. Convert Path objects
to strings before passing to repository methods.
"""
def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
"""Initialize with session maker."""
super().__init__(session_maker, Entity)
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
"""Get entity by permalink."""
"""Get entity by permalink.
Args:
permalink: Unique identifier for the entity
"""
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
return await self.find_one(query)
async def get_by_title(self, title: str) -> Optional[Entity]:
"""Get entity by title."""
"""Get entity by title.
Args:
title: Title of the entity to find
"""
query = self.select().where(Entity.title == title).options(*self.get_load_options())
return await self.find_one(query)
async def get_by_file_path(self, file_path: str) -> Optional[Entity]:
"""Get entity by file_path."""
query = self.select().where(Entity.file_path == file_path).options(*self.get_load_options())
return await self.find_one(query)
async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
"""Get entity by file_path.
async def list_entities(
self,
entity_type: Optional[str] = None,
sort_by: Optional[str] = "updated_at",
include_related: bool = False,
) -> Sequence[Entity]:
"""List all entities, optionally filtered by type and sorted."""
query = self.select()
# Always load base relations
query = query.options(*self.get_load_options())
# Apply filters
if entity_type:
# When include_related is True, get both:
# 1. Entities of the requested type
# 2. Entities that have relations with entities of the requested type
if include_related:
query = query.where(
or_(
Entity.entity_type == entity_type,
Entity.outgoing_relations.any(
Relation.to_entity.has(entity_type=entity_type)
),
Entity.incoming_relations.any(
Relation.from_entity.has(entity_type=entity_type)
),
)
)
else:
query = query.where(Entity.entity_type == entity_type)
# Apply sorting
if sort_by:
sort_field = getattr(Entity, sort_by, Entity.updated_at)
query = query.order_by(asc(sort_field))
result = await self.execute_query(query)
return list(result.scalars().all())
async def get_entity_types(self) -> List[str]:
"""Get list of distinct entity types."""
query = select(Entity.entity_type).distinct()
result = await self.execute_query(query, use_query_options=False)
return list(result.scalars().all())
async def search(self, query_str: str) -> List[Entity]:
Args:
file_path: Path to the entity file (will be converted to string internally)
"""
Search for entities.
Searches across:
- Entity names
- Entity types
- Entity descriptions
- Associated Observations content
"""
search_term = f"%{query_str}%"
query = (
self.select()
.where(
or_(
Entity.title.ilike(search_term),
Entity.entity_type.ilike(search_term),
Entity.summary.ilike(search_term),
Entity.observations.any(Observation.content.ilike(search_term)),
)
)
.where(Entity.file_path == str(file_path))
.options(*self.get_load_options())
)
result = await self.execute_query(query)
return list(result.scalars().all())
return await self.find_one(query)
async def delete_entities_by_doc_id(self, doc_id: int) -> bool:
"""Delete all entities associated with a document."""
return await self.delete_by_fields(doc_id=doc_id)
async def delete_by_file_path(self, file_path: Union[Path, str]) -> bool:
"""Delete entity with the provided file_path.
async def delete_by_file_path(self, file_path: str) -> bool:
"""Delete entity with the provided file_path."""
return await self.delete_by_fields(file_path=file_path)
Args:
file_path: Path to the entity file (will be converted to string internally)
"""
return await self.delete_by_fields(file_path=str(file_path))
def get_load_options(self) -> List[LoaderOption]:
"""Get SQLAlchemy loader options for eager loading relationships."""
return [
selectinload(Entity.observations).selectinload(Observation.entity),
# Load from_relations and both entities for each relation
@@ -126,8 +74,11 @@ class EntityRepository(Repository[Entity]):
]
async def find_by_permalinks(self, permalinks: List[str]) -> Sequence[Entity]:
"""Find multiple entities by their permalink."""
"""Find multiple entities by their permalink.
Args:
permalinks: List of permalink strings to find
"""
# Handle empty input explicitly
if not permalinks:
return []
@@ -139,18 +90,3 @@ class EntityRepository(Repository[Entity]):
result = await self.execute_query(query)
return list(result.scalars().all())
async def delete_by_permalinks(self, permalinks: List[str]) -> int:
"""Delete multiple entities by permalink."""
# Handle empty input explicitly
if not permalinks:
return 0
# Find matching entities
entities = await self.find_by_permalinks(permalinks)
if not entities:
return 0
# Use existing delete_by_ids
return await self.delete_by_ids([entity.id for entity in entities])
@@ -40,12 +40,6 @@ class RelationRepository(Repository[Relation]):
)
return await self.find_one(query)
async def find_by_entity(self, from_entity_id: int) -> Sequence[Relation]:
"""Find all relations from a specific entity."""
query = select(Relation).filter(Relation.from_id == from_entity_id)
result = await self.execute_query(query)
return result.scalars().all()
async def find_by_entities(self, from_id: int, to_id: int) -> Sequence[Relation]:
"""Find all relations between two entities."""
query = select(Relation).where((Relation.from_id == from_id) & (Relation.to_id == to_id))
@@ -73,6 +67,5 @@ class RelationRepository(Repository[Relation]):
result = await self.execute_query(query)
return result.scalars().all()
def get_load_options(self) -> List[LoaderOption]:
return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)]
+2 -39
View File
@@ -1,6 +1,5 @@
"""Base repository implementation."""
from datetime import datetime
from typing import Type, Optional, Any, Sequence, TypeVar, List
from loguru import logger
@@ -98,13 +97,6 @@ class Repository[T: Base]:
entities = (self.Model,)
return select(*entities)
async def refresh(self, instance: T, relationships: list[str] | None = None) -> None:
"""Refresh instance and optionally specified relationships."""
logger.debug(f"Refreshing {self.Model.__name__} instance: {getattr(instance, 'id', None)}")
async with db.scoped_session(self.session_maker) as session:
await session.refresh(instance, relationships or [])
logger.debug(f"Refreshed relationships: {relationships}")
async def find_all(self, skip: int = 0, limit: Optional[int] = 0) -> Sequence[T]:
"""Fetch records from the database with pagination."""
logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})")
@@ -149,35 +141,6 @@ class Repository[T: Base]:
logger.debug(f"No {self.Model.__name__} found")
return entity
async def find_modified_since(self, since: datetime) -> Sequence[T]:
"""Find all records modified since the given timestamp.
This method assumes the model has an updated_at column. Override
in subclasses if a different column should be used.
Args:
since: Datetime to search from
Returns:
Sequence of records modified since the timestamp
"""
logger.debug(f"Finding {self.Model.__name__} modified since: {since}")
if not hasattr(self.Model, "updated_at"):
raise AttributeError(f"{self.Model.__name__} does not have updated_at column")
query = (
select(self.Model)
.filter(self.Model.updated_at >= since)
.options(*self.get_load_options())
)
async with db.scoped_session(self.session_maker) as session:
result = await session.execute(query)
items = result.scalars().all()
logger.debug(f"Found {len(items)} modified {self.Model.__name__} records")
return items
async def create(self, data: dict) -> T:
"""Create a new record from a model instance."""
logger.debug(f"Creating {self.Model.__name__} from entity_data: {data}")
@@ -223,11 +186,11 @@ class Repository[T: Base]:
for key, value in entity_data.items():
if key in self.valid_columns:
setattr(entity, key, value)
elif isinstance(entity_data, self.Model):
for column in self.Model.__table__.columns.keys():
setattr(entity, column, getattr(entity_data, column))
await session.flush() # Make sure changes are flushed
await session.refresh(entity) # Refresh
@@ -21,6 +21,8 @@ class SearchIndexRow:
id: int
type: str
permalink: str
file_path: str
metadata: Optional[dict] = None
# date values
@@ -30,13 +32,9 @@ class SearchIndexRow:
# assigned in result
score: Optional[float] = None
# Common fields
permalink: Optional[str] = None
file_path: Optional[str] = None
# Type-specific fields
title: Optional[int] = None # entity
content: Optional[int] = None # entity, observation
title: Optional[str] = None # entity
content: Optional[str] = None # entity, observation
entity_id: Optional[int] = None # observations
category: Optional[str] = None # observations
from_id: Optional[int] = None # relations
@@ -70,7 +68,7 @@ class SearchRepository:
async def init_search_index(self):
"""Create or recreate the search index."""
logger.info("Initializing search index")
async with db.scoped_session(self.session_maker) as session:
await session.execute(CREATE_SEARCH_INDEX)
@@ -81,8 +79,8 @@ class SearchRepository:
For FTS5, special characters and phrases need to be quoted to be treated as a single token.
"""
# List of special characters that need quoting
special_chars = ['/', '*', '-', '.', ' ', '(', ')', '[', ']', '"', "'"]
special_chars = ["/", "*", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
# Check if term contains any special characters
if any(c in term for c in special_chars):
# If the term already contains quotes, escape them
@@ -96,16 +94,16 @@ class SearchRepository:
permalink: Optional[str] = None,
permalink_match: Optional[str] = None,
title: Optional[str] = None,
types: List[SearchItemType] = None,
types: Optional[List[SearchItemType]] = None,
after_date: Optional[datetime] = None,
entity_types: List[str] = None,
entity_types: Optional[List[str]] = None,
limit: int = 10,
) -> List[SearchIndexRow]:
"""Search across all indexed content with fuzzy matching."""
conditions = []
params = {}
order_by_clause = ""
# Handle text search for title and content
if search_text:
search_text = self._quote_search_term(search_text.lower().strip())
@@ -127,7 +125,7 @@ class SearchRepository:
if permalink_match:
params["permalink"] = self._quote_search_term(permalink_match)
conditions.append("permalink MATCH :permalink")
# Handle type filter
if types:
type_list = ", ".join(f"'{t.value}'" for t in types)
@@ -142,13 +140,13 @@ class SearchRepository:
if after_date:
params["after_date"] = after_date
conditions.append("datetime(created_at) > datetime(:after_date)")
# order by most recent first
order_by_clause = ", updated_at DESC"
# set limit on search query
params["limit"] = limit
# Build WHERE clause
where_clause = " AND ".join(conditions) if conditions else "1=1"
@@ -175,7 +173,7 @@ class SearchRepository:
LIMIT :limit
"""
#logger.debug(f"Search {sql} params: {params}")
# logger.debug(f"Search {sql} params: {params}")
async with db.scoped_session(self.session_maker) as session:
result = await session.execute(text(sql), params)
rows = result.fetchall()
@@ -201,9 +199,9 @@ class SearchRepository:
for row in rows
]
#for r in results:
# for r in results:
# logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
return results
async def index_item(
@@ -250,17 +248,14 @@ class SearchRepository:
async def execute_query(
self,
query: Executable,
params: Optional[Dict[str, Any]] = None,
params: Dict[str, Any],
) -> Result[Any]:
"""Execute a query asynchronously."""
#logger.debug(f"Executing query: {query}")
# logger.debug(f"Executing query: {query}, params: {params}")
async with db.scoped_session(self.session_maker) as session:
start_time = time.perf_counter()
if params:
result = await session.execute(query, params)
else:
result = await session.execute(query)
result = await session.execute(query, params)
end_time = time.perf_counter()
elapsed_time = end_time - start_time
logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
return result
return result
+4 -4
View File
@@ -23,7 +23,7 @@ from basic_memory.schemas.delete import (
from basic_memory.schemas.request import (
SearchNodesRequest,
GetEntitiesRequest,
CreateRelationsRequest, UpdateEntityRequest,
CreateRelationsRequest,
)
# Response models
@@ -40,7 +40,8 @@ from basic_memory.schemas.response import (
# Discovery and analytics models
from basic_memory.schemas.discovery import (
EntityTypeList,
ObservationCategoryList, TypedEntityList,
ObservationCategoryList,
TypedEntityList,
)
# For convenient imports, export all models
@@ -55,7 +56,6 @@ __all__ = [
"SearchNodesRequest",
"GetEntitiesRequest",
"CreateRelationsRequest",
"UpdateEntityRequest",
# Responses
"SQLAlchemyModel",
"ObservationResponse",
@@ -69,5 +69,5 @@ __all__ = [
# Discovery and Analytics
"EntityTypeList",
"ObservationCategoryList",
"TypedEntityList"
"TypedEntityList",
]
+21 -62
View File
@@ -14,14 +14,13 @@ Key Concepts:
import mimetypes
import re
from datetime import datetime
from enum import Enum
from pathlib import Path
from typing import List, Optional, Annotated, Dict
from annotated_types import MinLen, MaxLen
from dateparser import parse
from pydantic import BaseModel, BeforeValidator, Field, model_validator, ValidationError
from pydantic import BaseModel, BeforeValidator, Field, model_validator
from basic_memory.utils import generate_permalink
@@ -47,44 +46,6 @@ def to_snake_case(name: str) -> str:
return s2.lower()
def validate_path_format(path: str) -> str:
"""Validate path has the correct format: not empty."""
if not path or not isinstance(path, str):
raise ValueError("Path must be a non-empty string")
return path
class ObservationCategory(str, Enum):
"""Categories for structuring observations.
Categories help organize knowledge and make it easier to find later:
- tech: Implementation details and technical notes
- design: Architecture decisions and patterns
- feature: User-facing capabilities
- note: General observations (default)
- issue: Problems or concerns
- todo: Future work items
Categories are case-insensitive for easier use.
"""
TECH = "tech"
DESIGN = "design"
FEATURE = "feature"
NOTE = "note"
ISSUE = "issue"
TODO = "todo"
@classmethod
def _missing_(cls, value: str) -> "ObservationCategory":
"""Handle case-insensitive lookup."""
try:
return cls(value.lower())
except ValueError:
return None
def validate_timeframe(timeframe: str) -> str:
"""Convert human readable timeframes to a duration relative to the current time."""
if not isinstance(timeframe, str):
@@ -110,12 +71,9 @@ def validate_timeframe(timeframe: str) -> str:
return f"{days}d"
TimeFrame = Annotated[
str,
BeforeValidator(validate_timeframe)
]
TimeFrame = Annotated[str, BeforeValidator(validate_timeframe)]
PathId = Annotated[str, BeforeValidator(validate_path_format)]
Permalink = Annotated[str, MinLen(1)]
"""Unique identifier in format '{path}/{normalized_name}'."""
@@ -149,14 +107,16 @@ ObservationStr = Annotated[
MaxLen(1000), # Keep reasonable length
]
class Observation(BaseModel):
"""A single observation with category, content, and optional context."""
category: ObservationCategory
category: Optional[str] = None
content: ObservationStr
tags: Optional[List[str]] = Field(default_factory=list)
context: Optional[str] = None
class Relation(BaseModel):
"""Represents a directed edge between entities in the knowledge graph.
@@ -165,8 +125,8 @@ class Relation(BaseModel):
or recipient entity.
"""
from_id: PathId
to_id: PathId
from_id: Permalink
to_id: Permalink
relation_type: RelationType
context: Optional[str] = None
@@ -181,7 +141,7 @@ class Entity(BaseModel):
- Optional relations to other entities
- Optional description for high-level overview
"""
# private field to override permalink
_permalink: Optional[str] = None
@@ -192,28 +152,27 @@ class Entity(BaseModel):
entity_metadata: Optional[Dict] = Field(default=None, description="Optional metadata")
content_type: ContentType = Field(
description="MIME type of the content (e.g. text/markdown, image/jpeg)",
examples=["text/markdown", "image/jpeg"], default="text/markdown"
examples=["text/markdown", "image/jpeg"],
default="text/markdown",
)
@property
def file_path(self):
"""Get the file path for this entity based on its permalink."""
return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
@property
def permalink(self) -> PathId:
def permalink(self) -> Permalink:
"""Get a url friendly path}."""
return self._permalink or generate_permalink(self.file_path)
@model_validator(mode="after")
@classmethod
def infer_content_type(cls, entity: "Entity") -> Dict | None:
"""Infer content_type from file_path if not provided."""
if not entity.content_type:
path = Path(entity.file_path)
def infer_content_type(self) -> "Entity": # pragma: no cover
if not self.content_type:
path = Path(self.file_path)
if not path.exists():
return None
mime_type, _ = mimetypes.guess_type(path.name)
entity.content_type = mime_type or "text/plain"
return entity
self.content_type = "text/plain"
else:
mime_type, _ = mimetypes.guess_type(path.name)
self.content_type = mime_type or "text/plain"
return self
+2 -3
View File
@@ -21,7 +21,7 @@ from typing import List, Annotated
from annotated_types import MinLen
from pydantic import BaseModel
from basic_memory.schemas.base import Relation, Observation, PathId
from basic_memory.schemas.base import Permalink
class DeleteEntitiesRequest(BaseModel):
@@ -34,5 +34,4 @@ class DeleteEntitiesRequest(BaseModel):
4. Deletes the corresponding markdown file
"""
permalinks: Annotated[List[PathId], MinLen(1)]
permalinks: Annotated[List[Permalink], MinLen(1)]
+4 -1
View File
@@ -8,18 +8,21 @@ from basic_memory.schemas.response import EntityResponse
class EntityTypeList(BaseModel):
"""List of unique entity types in the system."""
types: List[str]
class ObservationCategoryList(BaseModel):
"""List of unique observation categories in the system."""
categories: List[str]
class TypedEntityList(BaseModel):
"""List of entities of a specific type."""
entity_type: str = Field(..., description="Type of entities in the list")
entities: List[EntityResponse]
total: int = Field(..., description="Total number of entities")
sort_by: Optional[str] = Field(None, description="Field used for sorting")
include_related: bool = Field(False, description="Whether related entities are included")
include_related: bool = Field(False, description="Whether related entities are included")
+12 -13
View File
@@ -1,11 +1,10 @@
"""Schemas for memory context."""
from datetime import datetime
from typing import Dict, List, Any, Optional, Annotated
from typing import List, Optional, Annotated, Sequence
import pydantic
from annotated_types import MinLen, MaxLen
from pydantic import BaseModel, field_validator, Field, BeforeValidator, TypeAdapter, AnyUrl
from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
from basic_memory.schemas.search import SearchItemType
@@ -32,19 +31,20 @@ def normalize_memory_url(url: str) -> str:
MemoryUrl = Annotated[
str,
BeforeValidator(str.strip), # Clean whitespace
MinLen(1),
MaxLen(2028),
MinLen(1),
MaxLen(2028),
]
memory_url = TypeAdapter(MemoryUrl)
memory_url = TypeAdapter(MemoryUrl)
def memory_url_path(url: memory_url) -> str:
def memory_url_path(url: memory_url) -> str: # pyright: ignore
"""
Returns the uri for a url value by removing the prefix "memory://" from a given MemoryUrl.
This function processes a given MemoryUrl by removing the "memory://"
prefix and returns the resulting string. If the provided url does not
begin with "memory://", the function will simply return the input url
prefix and returns the resulting string. If the provided url does not
begin with "memory://", the function will simply return the input url
unchanged.
:param url: A MemoryUrl object representing the URL with a "memory://" prefix.
@@ -55,7 +55,6 @@ def memory_url_path(url: memory_url) -> str:
return url.removeprefix("memory://")
class EntitySummary(BaseModel):
"""Simplified entity representation."""
@@ -101,14 +100,14 @@ class GraphContext(BaseModel):
"""Complete context response."""
# Direct matches
primary_results: List[EntitySummary | RelationSummary | ObservationSummary] = Field(
primary_results: Sequence[EntitySummary | RelationSummary | ObservationSummary] = Field(
description="results directly matching URI"
)
# Related entities
related_results: List[EntitySummary | RelationSummary | ObservationSummary] = Field(
related_results: Sequence[EntitySummary | RelationSummary | ObservationSummary] = Field(
description="related results"
)
# Context metadata
metadata: MemoryMetadata
metadata: MemoryMetadata
+4 -23
View File
@@ -1,23 +1,16 @@
"""Request schemas for interacting with the knowledge graph."""
from typing import List, Optional, Annotated, Dict, Any
from typing import List, Optional, Annotated
from annotated_types import MaxLen, MinLen
from pydantic import BaseModel
from basic_memory.schemas.base import (
Observation,
Entity,
Relation,
PathId,
ObservationCategory,
EntityType,
Permalink,
)
class SearchNodesRequest(BaseModel):
"""Search for entities in the knowledge graph.
@@ -47,7 +40,7 @@ class SearchNodesRequest(BaseModel):
"""
query: Annotated[str, MinLen(1), MaxLen(200)]
category: Optional[ObservationCategory] = None
category: Optional[str] = None
class GetEntitiesRequest(BaseModel):
@@ -58,20 +51,8 @@ class GetEntitiesRequest(BaseModel):
discovered through search.
"""
permalinks: Annotated[List[PathId], MinLen(1)]
permalinks: Annotated[List[Permalink], MinLen(1)]
class CreateRelationsRequest(BaseModel):
relations: List[Relation]
## update
# TODO remove UpdateEntityRequest
class UpdateEntityRequest(BaseModel):
"""Request to update an existing entity."""
title: Optional[str] = None
entity_type: Optional[EntityType] = None
content: Optional[str] = None
entity_metadata: Optional[Dict[str, Any]] = None
+10 -9
View File
@@ -15,7 +15,7 @@ from typing import List, Optional, Dict
from pydantic import BaseModel, ConfigDict, Field, AliasPath, AliasChoices
from basic_memory.schemas.base import Relation, PathId, EntityType, ContentType, Observation
from basic_memory.schemas.base import Relation, Permalink, EntityType, ContentType, Observation
class SQLAlchemyModel(BaseModel):
@@ -28,7 +28,7 @@ class SQLAlchemyModel(BaseModel):
model_config = ConfigDict(from_attributes=True)
class ObservationResponse(Observation, SQLAlchemyModel):
"""Schema for observation data returned from the service.
@@ -59,7 +59,7 @@ class RelationResponse(Relation, SQLAlchemyModel):
}
"""
from_id: PathId = Field(
from_id: Permalink = Field(
# use the permalink from the associated Entity
# or the from_id value
validation_alias=AliasChoices(
@@ -67,25 +67,26 @@ class RelationResponse(Relation, SQLAlchemyModel):
"from_id",
)
)
to_id: Optional[PathId] = Field(
to_id: Optional[Permalink] = Field( # pyright: ignore
# use the permalink from the associated Entity
# or the to_id value
validation_alias=AliasChoices(
AliasPath("to_entity", "permalink"),
"to_id",
), default=None
),
default=None,
)
to_name: Optional[PathId] = Field(
to_name: Optional[Permalink] = Field(
# use the permalink from the associated Entity
# or the to_id value
validation_alias=AliasChoices(
AliasPath("to_entity", "title"),
"to_name",
), default=None
),
default=None,
)
class EntityResponse(SQLAlchemyModel):
"""Complete entity data returned from the service.
@@ -125,7 +126,7 @@ class EntityResponse(SQLAlchemyModel):
}
"""
permalink: PathId
permalink: Permalink
title: str
file_path: str
entity_type: EntityType
+4 -7
View File
@@ -49,8 +49,6 @@ class SearchQuery(BaseModel):
@classmethod
def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]:
"""Convert datetime to ISO format if needed."""
if v is None:
return None
if isinstance(v, datetime):
return v.isoformat()
return v
@@ -71,12 +69,11 @@ class SearchResult(BaseModel):
id: int
type: SearchItemType
score: Optional[float] = None
metadata: Optional[dict] = None
score: float
permalink: str
file_path: str
# Common fields
permalink: Optional[str] = None
file_path: Optional[str] = None
metadata: Optional[dict] = None
# Type-specific fields
entity_id: Optional[int] = None # For observations
+2 -7
View File
@@ -1,12 +1,7 @@
"""Services package."""
from .database_service import DatabaseService
from .service import BaseService
from .file_service import FileService
from .entity_service import EntityService
__all__ = [
"BaseService",
"FileService",
"EntityService",
"DatabaseService"
]
__all__ = ["BaseService", "FileService", "EntityService"]
+116 -110
View File
@@ -50,8 +50,8 @@ class ContextService:
async def build_context(
self,
memory_url: MemoryUrl = None,
types: List[SearchItemType] = None,
memory_url: Optional[MemoryUrl] = None,
types: Optional[List[SearchItemType]] = None,
depth: int = 1,
since: Optional[datetime] = None,
max_results: int = 10,
@@ -66,9 +66,7 @@ class ContextService:
# Pattern matching - use search
if "*" in path:
logger.debug(f"Pattern search for '{path}'")
primary = await self.search_repository.search(
permalink_match=path
)
primary = await self.search_repository.search(permalink_match=path)
# Direct lookup for exact path
else:
@@ -120,11 +118,19 @@ class ContextService:
- Connected entities
- Their observations
- Relations that connect them
Note on depth:
Each traversal step requires two depth levels - one to find the relation,
and another to follow that relation to an entity. So a max_depth of 4 allows
traversal through two entities (relation->entity->relation->entity), while reaching
an entity three steps away requires max_depth=6 (relation->entity->relation->entity->relation->entity).
"""
max_depth = max_depth * 2
if not type_id_pairs:
return []
logger.debug(f"Finding connected items for {len(type_id_pairs)} with depth {max_depth}")
logger.debug(f"Finding connected items for {type_id_pairs} with depth {max_depth}")
# Build the VALUES clause directly since SQLite doesn't handle parameterized IN well
values = ", ".join([f"('{t}', {i})" for t, i in type_id_pairs])
@@ -132,7 +138,7 @@ class ContextService:
# Parameters for bindings
params = {"max_depth": max_depth, "max_results": max_results}
if since:
params["since_date"] = since.isoformat()
params["since_date"] = since.isoformat() # pyright: ignore
# Build date filter
date_filter = "AND base.created_at >= :since_date" if since else ""
@@ -140,113 +146,113 @@ class ContextService:
related_date_filter = "AND e.created_at >= :since_date" if since else ""
query = text(f"""
WITH RECURSIVE context_graph AS (
-- Base case: seed items (unchanged)
SELECT
id,
type,
title,
permalink,
file_path,
from_id,
to_id,
relation_type,
content,
category,
entity_id,
0 as depth,
id as root_id,
created_at,
created_at as relation_date,
0 as is_incoming
FROM search_index base
WHERE (base.type, base.id) IN ({values})
{date_filter}
WITH RECURSIVE context_graph AS (
-- Base case: seed items
SELECT
id,
type,
title,
permalink,
file_path,
from_id,
to_id,
relation_type,
content,
category,
entity_id,
0 as depth,
id as root_id,
created_at,
created_at as relation_date,
0 as is_incoming
FROM search_index base
WHERE (base.type, base.id) IN ({values})
{date_filter}
UNION -- Changed from UNION ALL
UNION ALL -- Allow same paths at different depths
-- Get relations from current entities
SELECT DISTINCT
r.id,
r.type,
r.title,
r.permalink,
r.file_path,
r.from_id,
r.to_id,
r.relation_type,
r.content,
r.category,
r.entity_id,
cg.depth + 1,
cg.root_id,
r.created_at,
r.created_at as relation_date,
CASE WHEN r.from_id = cg.id THEN 0 ELSE 1 END as is_incoming
FROM context_graph cg
JOIN search_index r ON (
cg.type = 'entity' AND
r.type = 'relation' AND
(r.from_id = cg.id OR r.to_id = cg.id)
{r1_date_filter}
)
WHERE cg.depth < :max_depth
-- Get relations from current entities
SELECT DISTINCT
r.id,
r.type,
r.title,
r.permalink,
r.file_path,
r.from_id,
r.to_id,
r.relation_type,
r.content,
r.category,
r.entity_id,
cg.depth + 1,
cg.root_id,
r.created_at,
r.created_at as relation_date,
CASE WHEN r.from_id = cg.id THEN 0 ELSE 1 END as is_incoming
FROM context_graph cg
JOIN search_index r ON (
cg.type = 'entity' AND
r.type = 'relation' AND
(r.from_id = cg.id OR r.to_id = cg.id)
{r1_date_filter}
)
WHERE cg.depth < :max_depth
UNION -- Changed from UNION ALL
UNION ALL
-- Get entities connected by relations
SELECT DISTINCT
e.id,
e.type,
e.title,
e.permalink,
e.file_path,
e.from_id,
e.to_id,
e.relation_type,
e.content,
e.category,
e.entity_id,
cg.depth,
cg.root_id,
e.created_at,
cg.relation_date,
cg.is_incoming
FROM context_graph cg
JOIN search_index e ON (
cg.type = 'relation' AND
e.type = 'entity' AND
e.id = CASE
WHEN cg.from_id = cg.id THEN cg.to_id
ELSE cg.from_id
END
{related_date_filter}
)
WHERE cg.depth < :max_depth
)
SELECT DISTINCT
type,
id,
title,
permalink,
file_path,
from_id,
to_id,
relation_type,
content,
category,
entity_id,
MIN(depth) as depth,
root_id,
created_at
FROM context_graph
WHERE (type, id) NOT IN ({values})
GROUP BY
type, id, title, permalink, from_id, to_id,
relation_type, category, entity_id,
root_id, created_at
ORDER BY depth, type, id
LIMIT :max_results
-- Get entities connected by relations
SELECT DISTINCT
e.id,
e.type,
e.title,
e.permalink,
e.file_path,
e.from_id,
e.to_id,
e.relation_type,
e.content,
e.category,
e.entity_id,
cg.depth + 1, -- Increment depth for entities
cg.root_id,
e.created_at,
cg.relation_date,
cg.is_incoming
FROM context_graph cg
JOIN search_index e ON (
cg.type = 'relation' AND
e.type = 'entity' AND
e.id = CASE
WHEN cg.is_incoming = 0 THEN cg.to_id -- Fixed entity lookup
ELSE cg.from_id
END
{related_date_filter}
)
WHERE cg.depth < :max_depth
)
SELECT DISTINCT
type,
id,
title,
permalink,
file_path,
from_id,
to_id,
relation_type,
content,
category,
entity_id,
MIN(depth) as depth,
root_id,
created_at
FROM context_graph
WHERE (type, id) NOT IN ({values})
GROUP BY
type, id, title, permalink, from_id, to_id,
relation_type, category, entity_id,
root_id, created_at
ORDER BY depth, type, id
LIMIT :max_results
""")
result = await self.search_repository.execute_query(query, params=params)
@@ -1,159 +0,0 @@
"""Service for managing database lifecycle and schema validation."""
from datetime import datetime
from pathlib import Path
from typing import Optional, Tuple, List
from alembic.runtime.migration import MigrationContext
from alembic.autogenerate import compare_metadata
from loguru import logger
from sqlalchemy import MetaData
from sqlalchemy.ext.asyncio import AsyncSession
from basic_memory import db
from basic_memory.config import ProjectConfig
from basic_memory.models import Base
from basic_memory.repository.search_repository import SearchRepository
async def check_schema_matches_models(session: AsyncSession) -> Tuple[bool, List[str]]:
"""Check if database schema matches SQLAlchemy models.
Returns:
tuple[bool, list[str]]: (matches, list of differences)
"""
# Get current DB schema via migration context
conn = await session.connection()
def _compare_schemas(connection):
context = MigrationContext.configure(connection)
return compare_metadata(context, Base.metadata)
# Run comparison in sync context
differences = await conn.run_sync(_compare_schemas)
if not differences:
return True, []
# Format differences into readable messages
diff_messages = []
for diff in differences:
if diff[0] == 'add_table':
diff_messages.append(f"Missing table: {diff[1].name}")
elif diff[0] == 'remove_table':
diff_messages.append(f"Extra table: {diff[1].name}")
elif diff[0] == 'add_column':
diff_messages.append(f"Missing column: {diff[3]} in table {diff[2]}")
elif diff[0] == 'remove_column':
diff_messages.append(f"Extra column: {diff[3]} in table {diff[2]}")
elif diff[0] == 'modify_type':
diff_messages.append(f"Column type mismatch: {diff[3]} in table {diff[2]}")
return False, diff_messages
class DatabaseService:
"""Manages database lifecycle including schema validation and backups."""
def __init__(
self,
config: ProjectConfig,
db_type: db.DatabaseType = db.DatabaseType.FILESYSTEM,
):
self.config = config
self.db_path = Path(config.database_path)
self.db_type = db_type
async def create_backup(self) -> Optional[Path]:
"""Create backup of existing database file.
Returns:
Optional[Path]: Path to backup file if created, None if no DB exists
"""
if self.db_type == db.DatabaseType.MEMORY:
return None # Skip backups for in-memory DB
if not self.db_path.exists():
return None
# Create backup with timestamp
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = self.db_path.with_suffix(f".{timestamp}.backup")
try:
self.db_path.rename(backup_path)
logger.info(f"Created database backup: {backup_path}")
# make a new empty file
self.db_path.touch()
return backup_path
except Exception as e:
logger.error(f"Failed to create database backup: {e}")
return None
async def initialize_db(self):
"""Initialize database with current schema."""
logger.info("Initializing database...")
if self.db_type == db.DatabaseType.FILESYSTEM:
await self.create_backup()
# Drop existing tables if any
await db.drop_db()
# Create tables with current schema
await db.get_or_create_db(
db_path=self.db_path,
db_type=self.db_type
)
logger.info("Database initialized with current schema")
async def check_db(self) -> bool:
"""Check database state and rebuild if schema doesn't match models.
Returns:
bool: True if DB is ready for use, False if initialization failed
"""
try:
_, session_maker = await db.get_or_create_db(
db_path=self.db_path,
db_type=self.db_type
)
async with db.scoped_session(session_maker) as db_session:
# Check actual schema matches
matches, differences = await check_schema_matches_models(db_session)
if not matches:
logger.warning("Database schema does not match models:")
for diff in differences:
logger.warning(f" {diff}")
logger.info("Rebuilding database to match current models...")
await self.initialize_db()
return True
logger.info("Database schema matches models")
return True
except Exception as e:
logger.error(f"Database initialization failed: {e}")
return False
async def cleanup_backups(self, keep_count: int = 5):
"""Clean up old database backups, keeping the N most recent."""
if self.db_type == db.DatabaseType.MEMORY:
return # Skip cleanup for in-memory DB
backup_pattern = "*.backup" # Use relative pattern
backups = sorted(
self.db_path.parent.glob(backup_pattern),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
# Remove old backups
for backup in backups[keep_count:]:
try:
backup.unlink()
logger.debug(f"Removed old backup: {backup}")
except Exception as e:
logger.error(f"Failed to remove backup {backup}: {e}")
+26 -63
View File
@@ -1,10 +1,9 @@
"""Service for managing entities in the database."""
from pathlib import Path
from typing import Sequence, List, Optional
from typing import Sequence, List, Optional, Tuple, Union
import frontmatter
from frontmatter import Post
from loguru import logger
from sqlalchemy.exc import IntegrityError
@@ -14,6 +13,7 @@ from basic_memory.models import Entity as EntityModel, Observation, Relation
from basic_memory.repository import ObservationRepository, RelationRepository
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.schemas import Entity as EntitySchema
from basic_memory.schemas.base import Permalink
from basic_memory.services.exceptions import EntityNotFoundError, EntityCreationError
from basic_memory.services import FileService
from basic_memory.services import BaseService
@@ -42,9 +42,7 @@ class EntityService(BaseService[EntityModel]):
self.link_resolver = link_resolver
async def resolve_permalink(
self,
file_path: Path,
markdown: Optional[EntityMarkdown] = None
self, file_path: Permalink | Path, markdown: Optional[EntityMarkdown] = None
) -> str:
"""Get or generate unique permalink for an entity.
@@ -54,19 +52,17 @@ class EntityService(BaseService[EntityModel]):
3. For existing files, keep current permalink from db
4. Generate new unique permalink from file path
"""
file_path = str(file_path)
# If markdown has explicit permalink, try to validate it
if markdown and markdown.frontmatter.permalink:
desired_permalink = markdown.frontmatter.permalink
existing = await self.repository.get_by_permalink(desired_permalink)
# If no conflict or it's our own file, use as is
if not existing or existing.file_path == file_path:
if not existing or existing.file_path == str(file_path):
return desired_permalink
# For existing files, try to find current permalink
existing = await self.repository.get_by_file_path(file_path)
existing = await self.repository.get_by_file_path(str(file_path))
if existing:
return existing.permalink
@@ -85,12 +81,11 @@ class EntityService(BaseService[EntityModel]):
logger.debug(f"creating unique permalink: {permalink}")
return permalink
async def create_or_update_entity(self, schema: EntitySchema) -> (EntityModel, bool):
"""Create new entity or update existing one.
if a new entity is created, the return value is (entity, True)
"""
async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]:
"""Create new entity or update existing one.
Returns: (entity, is_new) where is_new is True if a new entity was created
"""
logger.debug(f"Creating or updating entity: {schema}")
# Try to find existing entity using smart resolution
@@ -107,7 +102,7 @@ class EntityService(BaseService[EntityModel]):
"""Create a new entity and write to filesystem."""
logger.debug(f"Creating entity: {schema.permalink}")
# get file path
# Get file path and ensure it's a Path object
file_path = Path(schema.file_path)
if await self.file_service.exists(file_path):
@@ -127,11 +122,9 @@ class EntityService(BaseService[EntityModel]):
# parse entity from file
entity_markdown = await self.entity_parser.parse_file(file_path)
# create entity
created_entity = await self.create_entity_from_markdown(
file_path, entity_markdown
)
await self.create_entity_from_markdown(file_path, entity_markdown)
# add relations
entity = await self.update_entity_relations(file_path, entity_markdown)
@@ -139,12 +132,11 @@ class EntityService(BaseService[EntityModel]):
# Set final checksum to mark complete
return await self.repository.update(entity.id, {"checksum": checksum})
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
"""Update an entity's content and metadata."""
logger.debug(f"Updating entity with permalink: {entity.permalink}")
# get file path
# Convert file path string to Path
file_path = Path(entity.file_path)
post = await schema_to_markdown(schema)
@@ -157,9 +149,7 @@ class EntityService(BaseService[EntityModel]):
entity_markdown = await self.entity_parser.parse_file(file_path)
# update entity in db
entity = await self.update_entity_and_observations(
file_path, entity_markdown
)
entity = await self.update_entity_and_observations(file_path, entity_markdown)
# add relations
await self.update_entity_relations(file_path, entity_markdown)
@@ -187,10 +177,6 @@ class EntityService(BaseService[EntityModel]):
logger.info(f"Entity not found: {permalink}")
return True # Already deleted
except Exception as e:
logger.error(f"Failed to delete entity: {e}")
raise
async def get_by_permalink(self, permalink: str) -> EntityModel:
"""Get entity by type and name combination."""
logger.debug(f"Getting entity by permalink: {permalink}")
@@ -199,32 +185,14 @@ class EntityService(BaseService[EntityModel]):
raise EntityNotFoundError(f"Entity not found: {permalink}")
return db_entity
async def get_all(self) -> Sequence[EntityModel]:
"""Get all entities."""
return await self.repository.find_all()
async def get_entity_types(self) -> List[str]:
"""Get list of all distinct entity types in the system."""
logger.debug("Getting all distinct entity types")
return await self.repository.get_entity_types()
async def list_entities(
self,
entity_type: Optional[str] = None,
sort_by: Optional[str] = "updated_at",
include_related: bool = False,
) -> Sequence[EntityModel]:
"""List entities with optional filtering and sorting."""
logger.debug(f"Listing entities: type={entity_type} sort={sort_by}")
return await self.repository.list_entities(entity_type=entity_type, sort_by=sort_by)
async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
"""Get specific nodes and their relationships."""
logger.debug(f"Getting entities permalinks: {permalinks}")
return await self.repository.find_by_permalinks(permalinks)
async def delete_entity_by_file_path(self, file_path):
await self.repository.delete_by_file_path(file_path)
async def delete_entity_by_file_path(self, file_path: Union[str, Path]) -> None:
"""Delete entity by file path."""
await self.repository.delete_by_file_path(str(file_path))
async def create_entity_from_markdown(
self, file_path: Path, markdown: EntityMarkdown
@@ -234,15 +202,15 @@ class EntityService(BaseService[EntityModel]):
Creates the entity with null checksum to indicate sync not complete.
Relations will be added in second pass.
"""
logger.debug(f"Creating entity: {markdown.frontmatter.title}")
logger.debug(f"Creating entity: {markdown.frontmatter.title}")
model = entity_model_from_markdown(file_path, markdown)
# Mark as incomplete sync
# Mark as incomplete because we still need to add relations
model.checksum = None
return await self.add(model)
return await self.repository.add(model)
async def update_entity_and_observations(
self, file_path: Path | str, markdown: EntityMarkdown
self, file_path: Path, markdown: EntityMarkdown
) -> EntityModel:
"""Update entity fields and observations.
@@ -250,11 +218,8 @@ class EntityService(BaseService[EntityModel]):
to indicate sync not complete.
"""
logger.debug(f"Updating entity and observations: {file_path}")
file_path = str(file_path)
db_entity = await self.repository.get_by_file_path(file_path)
if not db_entity:
raise EntityNotFoundError(f"Entity not found: {file_path}")
db_entity = await self.repository.get_by_file_path(str(file_path))
# Clear observations for entity
await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
@@ -277,9 +242,8 @@ class EntityService(BaseService[EntityModel]):
# checksum value is None == not finished with sync
db_entity.checksum = None
# update entity
# checksum value is None == not finished with sync
return await self.repository.update(
db_entity.id,
db_entity,
@@ -287,14 +251,13 @@ class EntityService(BaseService[EntityModel]):
async def update_entity_relations(
self,
file_path: Path | str,
file_path: Path,
markdown: EntityMarkdown,
) -> EntityModel:
"""Update relations for entity"""
logger.debug(f"Updating relations for entity: {file_path}")
file_path = str(file_path)
db_entity = await self.repository.get_by_file_path(file_path)
db_entity = await self.repository.get_by_file_path(str(file_path))
# Clear existing relations first
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
@@ -328,4 +291,4 @@ class EntityService(BaseService[EntityModel]):
)
continue
return await self.repository.get_by_file_path(file_path)
return await self.repository.get_by_file_path(str(file_path))
+1
View File
@@ -9,6 +9,7 @@ class EntityNotFoundError(Exception):
pass
class EntityCreationError(Exception):
"""Raised when an entity cannot be created"""
+75 -111
View File
@@ -1,20 +1,23 @@
"""Service for file operations with checksum tracking."""
from pathlib import Path
from typing import Optional, Tuple
from typing import Tuple, Union
from loguru import logger
from basic_memory import file_utils
from basic_memory.markdown.markdown_processor import MarkdownProcessor
from basic_memory.markdown.utils import entity_model_to_markdown
from basic_memory.models import Entity as EntityModel
from basic_memory.services.exceptions import FileOperationError
from basic_memory.schemas import Entity as EntitySchema
from basic_memory.services.exceptions import FileOperationError
class FileService:
"""
Service for handling file operations.
"""Service for handling file operations.
All paths are handled as Path objects internally. Strings are converted to
Path objects when passed in. Relative paths are assumed to be relative to
base_path.
Features:
- Consistent file writing with checksums
@@ -28,105 +31,66 @@ class FileService:
base_path: Path,
markdown_processor: MarkdownProcessor,
):
self.base_path = base_path
self.base_path = base_path.resolve() # Get absolute path
self.markdown_processor = markdown_processor
def get_entity_path(self, entity: EntityModel| EntitySchema) -> Path:
"""Generate absolute filesystem path for entity."""
return self.base_path / f"{entity.file_path}"
async def write_entity_file(
self,
entity: EntityModel,
content: Optional[str] = None,
expected_checksum: Optional[str] = None,
) -> Tuple[Path, str]:
"""Write entity to filesystem and return path and checksum.
Uses read->modify->write pattern:
1. Read existing file if it exists
2. Update with new content if provided
3. Write back atomically
def get_entity_path(self, entity: Union[EntityModel, EntitySchema]) -> Path:
"""Generate absolute filesystem path for entity.
Args:
entity: Entity model to write
content: Optional new content (preserves existing if None)
expected_checksum: Optional checksum to verify file hasn't changed
entity: Entity model or schema with file_path attribute
Returns:
Tuple of (file path, new checksum)
Raises:
FileOperationError: If write fails
Absolute Path to the entity file
"""
try:
path = self.get_entity_path(entity)
# Read current state if file exists
if path.exists():
# read the existing file
existing_markdown = await self.markdown_processor.read_file(path)
# if content is supplied use it or existing content
content=content or existing_markdown.content
# Create new file structure with provided content
markdown = entity_model_to_markdown(entity, content=content)
# Write back atomically
checksum = await self.markdown_processor.write_file(
path=path, markdown=markdown, expected_checksum=expected_checksum
)
return path, checksum
except Exception as e:
logger.exception(f"Failed to write entity file: {e}")
raise FileOperationError(f"Failed to write entity file: {e}")
return self.base_path / entity.file_path
async def read_entity_content(self, entity: EntityModel) -> str:
"""Get entity's content without frontmatter or structured sections (used to index for search)
"""Get entity's content without frontmatter or structured sections.
Used to index for search. Returns raw content without frontmatter,
observations, or relations.
Args:
entity: Entity to read content for
Returns:
Raw content without frontmatter, observations, or relations
Raises:
FileOperationError: If entity file doesn't exist
Raw content string without metadata sections
"""
logger.debug(f"Reading entity with permalink: {entity.permalink}")
try:
file_path = self.get_entity_path(entity)
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content or ""
except Exception as e:
logger.error(f"Failed to read entity content: {e}")
raise FileOperationError(f"Failed to read entity content: {e}")
file_path = self.get_entity_path(entity)
markdown = await self.markdown_processor.read_file(file_path)
return markdown.content or ""
async def delete_entity_file(self, entity: EntityModel) -> None:
"""Delete entity file from filesystem."""
try:
path = self.get_entity_path(entity)
await self.delete_file(path)
except Exception as e:
logger.error(f"Failed to delete entity file: {e}")
raise FileOperationError(f"Failed to delete entity file: {e}")
async def exists(self, path: Path) -> bool:
"""
Check if file exists at the provided path. If path is relative, it is assumed to be relative to base_path.
"""Delete entity file from filesystem.
Args:
path: Path to check
entity: Entity model whose file should be deleted
Raises:
FileOperationError: If deletion fails
"""
path = self.get_entity_path(entity)
await self.delete_file(path)
async def exists(self, path: Union[Path, str]) -> bool:
"""Check if file exists at the provided path.
If path is relative, it is assumed to be relative to base_path.
Args:
path: Path to check (Path object or string)
Returns:
True if file exists, False otherwise
Raises:
FileOperationError: If check fails
"""
try:
path = Path(path)
if path.is_absolute():
return path.exists()
else:
@@ -135,12 +99,14 @@ class FileService:
logger.error(f"Failed to check file existence {path}: {e}")
raise FileOperationError(f"Failed to check file existence: {e}")
async def write_file(self, path: Path, content: str) -> str:
"""
Write content to file and return checksum.
async def write_file(self, path: Union[Path, str], content: str) -> str:
"""Write content to file and return checksum.
Handles both absolute and relative paths. Relative paths are resolved
against base_path.
Args:
path: Path where to write
path: Where to write (Path object or string)
content: Content to write
Returns:
@@ -149,30 +115,33 @@ class FileService:
Raises:
FileOperationError: If write fails
"""
path = path if path.is_absolute() else self.base_path / path
path = Path(path)
full_path = path if path.is_absolute() else self.base_path / path
try:
# Ensure parent directory exists
await file_utils.ensure_directory(path.parent)
await file_utils.ensure_directory(full_path.parent)
# Write content atomically
await file_utils.write_file_atomic(path, content)
await file_utils.write_file_atomic(full_path, content)
# Compute and return checksum
checksum = await file_utils.compute_checksum(content)
logger.debug(f"wrote file: {path}, checksum: {checksum} content: \n{content}")
logger.debug(f"wrote file: {full_path}, checksum: {checksum}")
return checksum
except Exception as e:
logger.error(f"Failed to write file {path}: {e}")
logger.error(f"Failed to write file {full_path}: {e}")
raise FileOperationError(f"Failed to write file: {e}")
async def read_file(self, path: Path) -> Tuple[str, str]:
"""
Read file and compute checksum.
async def read_file(self, path: Union[Path, str]) -> Tuple[str, str]:
"""Read file and compute checksum.
Handles both absolute and relative paths. Relative paths are resolved
against base_path.
Args:
path: Path to read
path: Path to read (Path object or string)
Returns:
Tuple of (content, checksum)
@@ -180,33 +149,28 @@ class FileService:
Raises:
FileOperationError: If read fails
"""
path = path if path.is_absolute() else self.base_path / path
path = Path(path)
full_path = path if path.is_absolute() else self.base_path / path
try:
content = path.read_text()
checksum = await file_utils.compute_checksum(content)
logger.debug(f"read file: {path}, checksum: {checksum}")
logger.debug(f"read file: {full_path}, checksum: {checksum}")
return content, checksum
except Exception as e:
logger.error(f"Failed to read file {path}: {e}")
logger.error(f"Failed to read file {full_path}: {e}")
raise FileOperationError(f"Failed to read file: {e}")
async def delete_file(self, path: Path) -> None:
"""
Delete file if it exists.
async def delete_file(self, path: Union[Path, str]) -> None:
"""Delete file if it exists.
Handles both absolute and relative paths. Relative paths are resolved
against base_path.
Args:
path: Path to delete
Raises:
FileOperationError: If deletion fails
path: Path to delete (Path object or string)
"""
path = path if path.is_absolute() else self.base_path / path
try:
path.unlink(missing_ok=True)
except Exception as e:
logger.error(f"Failed to delete file {path}: {e}")
raise FileOperationError(f"Failed to delete file: {e}")
def path(self, path_string: str, absolute: bool = False):
return Path( self.base_path / path_string ) if absolute else Path(path_string).relative_to(self.base_path)
path = Path(path)
full_path = path if path.is_absolute() else self.base_path / path
full_path.unlink(missing_ok=True)
+9 -9
View File
@@ -5,9 +5,10 @@ from typing import Optional, Tuple, List
from loguru import logger
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.repository.search_repository import SearchIndexRow
from basic_memory.services.search_service import SearchService
from basic_memory.models import Entity
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
from basic_memory.schemas.search import SearchQuery, SearchItemType
class LinkResolver:
@@ -45,18 +46,19 @@ class LinkResolver:
return entity
if use_search:
# 3. Fall back to search for fuzzy matching on title if specified
results = await self.search_service.search(
query=SearchQuery(title=clean_text, types=[SearchItemType.ENTITY]),
)
if results:
# Look for best match
best_match = self._select_best_match(clean_text, results)
logger.debug(f"Selected best match from {len(results)} results: {best_match.permalink}")
logger.debug(
f"Selected best match from {len(results)} results: {best_match.permalink}"
)
return await self.entity_repository.get_by_permalink(best_match.permalink)
# if we couldn't find anything then return None
return None
@@ -85,7 +87,7 @@ class LinkResolver:
return text, alias
def _select_best_match(self, search_text: str, results: List[SearchResult]) -> Entity:
def _select_best_match(self, search_text: str, results: List[SearchIndexRow]) -> Entity:
"""Select best match from search results.
Uses multiple criteria:
@@ -93,9 +95,6 @@ class LinkResolver:
2. Word matches in path
3. Overall search score
"""
if not results:
raise ValueError("Cannot select from empty results")
# Get search terms for matching
terms = search_text.lower().split()
@@ -104,6 +103,7 @@ class LinkResolver:
for result in results:
# Start with base score (lower is better)
score = result.score
assert score is not None
# Parse path components
path_parts = result.permalink.lower().split("/")
+22 -15
View File
@@ -1,5 +1,6 @@
"""Service for search operations."""
from datetime import datetime
from typing import List, Optional, Set
from fastapi import BackgroundTasks
@@ -8,9 +9,8 @@ from loguru import logger
from basic_memory.models import Entity
from basic_memory.repository import EntityRepository
from basic_memory.repository.search_repository import SearchRepository, SearchIndexRow
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
from basic_memory.schemas.search import SearchQuery, SearchItemType
from basic_memory.services import FileService
from basic_memory.services.exceptions import FileOperationError
class SearchService:
@@ -51,9 +51,7 @@ class SearchService:
logger.info("Reindex complete")
async def search(
self, query: SearchQuery, context: Optional[List[str]] = None
) -> List[SearchResult]:
async def search(self, query: SearchQuery) -> List[SearchIndexRow]:
"""Search across all indexed content.
Supports three modes:
@@ -67,6 +65,16 @@ class SearchService:
logger.debug(f"Searching with query: {query}")
after_date = (
(
query.after_date
if isinstance(query.after_date, datetime)
else datetime.fromisoformat(query.after_date)
)
if query.after_date
else None
)
# permalink search
results = await self.repository.search(
search_text=query.text,
@@ -75,12 +83,13 @@ class SearchService:
title=query.title,
types=query.types,
entity_types=query.entity_types,
after_date=query.after_date,
after_date=after_date,
)
return results
def _generate_variants(self, text: str) -> Set[str]:
@staticmethod
def _generate_variants(text: str) -> Set[str]:
"""Generate text variants for better fuzzy matching.
Creates variations of the text to improve match chances:
@@ -140,7 +149,6 @@ class SearchService:
title_variants = self._generate_variants(entity.title)
content_parts.extend(title_variants)
# TODO should we do something to content on indexing?
content = await self.file_service.read_entity_content(entity)
if content:
content_parts.append(content)
@@ -162,8 +170,8 @@ class SearchService:
metadata={
"entity_type": entity.entity_type,
},
created_at=entity.created_at.isoformat(),
updated_at=entity.updated_at.isoformat(),
created_at=entity.created_at,
updated_at=entity.updated_at,
)
)
@@ -183,8 +191,8 @@ class SearchService:
metadata={
"tags": obs.tags,
},
created_at=entity.created_at.isoformat(),
updated_at=entity.updated_at.isoformat(),
created_at=entity.created_at,
updated_at=entity.updated_at,
)
)
@@ -201,15 +209,14 @@ class SearchService:
SearchIndexRow(
id=rel.id,
title=relation_title,
content=rel.context or "",
permalink=rel.permalink,
file_path=entity.file_path,
type=SearchItemType.RELATION.value,
from_id=rel.from_id,
to_id=rel.to_id,
relation_type=rel.relation_type,
created_at=entity.created_at.isoformat(),
updated_at=entity.updated_at.isoformat(),
created_at=entity.created_at,
updated_at=entity.updated_at,
)
)
+3 -24
View File
@@ -1,36 +1,15 @@
"""Base service class."""
from datetime import datetime
from typing import TypeVar, Generic, List, Sequence
from typing import TypeVar, Generic
from basic_memory.models import Base
from basic_memory.repository.repository import Repository
T = TypeVar("T", bound=Base)
R = TypeVar("R", bound=Repository)
class BaseService(Generic[T]):
"""Base service that takes a repository."""
def __init__(self, repository: R):
def __init__(self, repository):
"""Initialize service with repository."""
self.repository = repository
async def add(self, model: T) -> T:
"""Add model to repository."""
return await self.repository.add(model)
async def add_all(self, models: List[T]) -> Sequence[T]:
"""Add a List of models to repository."""
return await self.repository.add_all(models)
async def get_modified_since(self, since: datetime) -> Sequence[T]:
"""Get all items modified since the given timestamp.
Args:
since: Datetime to search from
Returns:
Sequence of items modified since the timestamp
"""
return await self.repository.find_modified_since(since)
+2 -2
View File
@@ -1,5 +1,5 @@
from .file_change_scanner import FileChangeScanner
from .sync_service import SyncService
from .watch_service import WatchService
__all__ = ["SyncService", "FileChangeScanner"]
__all__ = ["SyncService", "FileChangeScanner", "WatchService"]
+3 -7
View File
@@ -69,11 +69,7 @@ class FileChangeScanner:
rel_path = str(path.relative_to(directory))
content = path.read_text()
checksum = await compute_checksum(content)
if checksum: # Only store valid checksums
result.files[rel_path] = checksum
else:
result.errors[rel_path] = "Failed to compute checksum"
result.files[rel_path] = checksum
except Exception as e:
rel_path = str(path.relative_to(directory))
@@ -134,7 +130,7 @@ class FileChangeScanner:
logger.debug(f" Moved: {len(report.moves)}")
logger.debug(f" Deleted: {len(report.deleted)}")
if scan_result.errors:
if scan_result.errors: # pragma: no cover
logger.warning("Files skipped due to errors:")
for file_path, error in scan_result.errors.items():
logger.warning(f" {file_path}: {error}")
@@ -151,7 +147,7 @@ class FileChangeScanner:
"""
return {
r.file_path: FileState(
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum or ""
)
for r in db_records
}
+35 -40
View File
@@ -59,9 +59,6 @@ class SyncService:
for permalink in permalinks:
await self.search_service.delete_by_permalink(permalink)
else:
logger.debug(f"No entity found to delete: {file_path}")
async def sync(self, directory: Path) -> SyncReport:
"""Sync knowledge files with database."""
changes = await self.scanner.find_knowledge_changes(directory)
@@ -77,69 +74,63 @@ class SyncService:
entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
)
# update search index
await self.search_service.index_entity(updated)
if updated:
await self.search_service.index_entity(updated)
# Handle deletions next
# remove rows from db for files no longer present
for file_path in changes.deleted:
await self.handle_entity_deletion(file_path)
for path in changes.deleted:
await self.handle_entity_deletion(path)
# Parse files that need updating
parsed_entities: Dict[str, EntityMarkdown] = {}
for file_path in [*changes.new, *changes.modified]:
entity_markdown = await self.entity_parser.parse_file(directory / file_path)
parsed_entities[file_path] = entity_markdown
for path in [*changes.new, *changes.modified]:
entity_markdown = await self.entity_parser.parse_file(directory / path)
parsed_entities[path] = entity_markdown
# First pass: Create/update entities
# entities will have a null checksum to indicate they are not complete
for file_path, entity_markdown in parsed_entities.items():
for path, entity_markdown in parsed_entities.items():
# Get unique permalink and update markdown if needed
permalink = await self.entity_service.resolve_permalink(
file_path,
markdown=entity_markdown
Path(path), markdown=entity_markdown
)
if permalink != entity_markdown.frontmatter.permalink:
# Add/update permalink in frontmatter
logger.info(f"Adding permalink '{permalink}' to file: {file_path}")
logger.info(f"Adding permalink '{permalink}' to file: {path}")
# update markdown
entity_markdown.frontmatter.metadata["permalink"] = permalink
# update file frontmatter
updated_checksum = await file_utils.update_frontmatter(
directory / file_path,
{"permalink": permalink}
directory / path, {"permalink": permalink}
)
# Update checksum in changes report since file was modified
changes.checksums[file_path] = updated_checksum
changes.checksums[path] = updated_checksum
# if the file is new, create an entity
if file_path in changes.new:
if path in changes.new:
# Create entity with final permalink
logger.debug(f"Creating new entity_markdown: {file_path}")
await self.entity_service.create_entity_from_markdown(
file_path, entity_markdown
)
logger.debug(f"Creating new entity_markdown: {path}")
await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown)
# otherwise we need to update the entity and observations
else:
logger.debug(f"Updating entity_markdown: {file_path}")
logger.debug(f"Updating entity_markdown: {path}")
await self.entity_service.update_entity_and_observations(
file_path, entity_markdown
Path(path), entity_markdown
)
# Second pass
for file_path, entity_markdown in parsed_entities.items():
logger.debug(f"Updating relations for: {file_path}")
for path, entity_markdown in parsed_entities.items():
logger.debug(f"Updating relations for: {path}")
# Process relations
checksum = changes.checksums[file_path]
entity = await self.entity_service.update_entity_relations(
file_path, entity_markdown
)
checksum = changes.checksums[path]
entity = await self.entity_service.update_entity_relations(Path(path), entity_markdown)
# add to search index
await self.search_service.index_entity(entity)
@@ -153,18 +144,22 @@ class SyncService:
target_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
# check we found a link that is not the source
if target_entity and target_entity.id != relation.from_id:
logger.debug(f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}")
logger.debug(
f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}"
)
try:
await self.relation_repository.update(relation.id, {
"to_id": target_entity.id,
"to_name": target_entity.title # Update to actual title
})
except IntegrityError as e:
logger.info(f"Ignoring duplicate relation {relation}")
await self.relation_repository.update(
relation.id,
{
"to_id": target_entity.id,
"to_name": target_entity.title, # Update to actual title
},
)
except IntegrityError:
logger.debug(f"Ignoring duplicate relation {relation}")
# update search index
await self.search_service.index_entity(target_entity)
return changes
+6 -38
View File
@@ -2,44 +2,15 @@
from dataclasses import dataclass, field
from typing import Set, Dict, Optional
from watchfiles import Change
from basic_memory.services.file_service import FileService
@dataclass
class FileChange:
"""A change to a file detected by the watch service.
Attributes:
change_type: Type of change (added, modified, deleted)
path: Path to the file
checksum: File checksum (None for deleted files)
"""
change_type: Change
path: str
checksum: Optional[str] = None
@classmethod
async def from_path(cls, path: str, change_type: Change, file_service: FileService) -> "FileChange":
"""Create FileChange from a path, computing checksum if file exists.
Args:
path: Path to the file
change_type: Type of change detected
file_service: Service to read file and compute checksum
Returns:
FileChange with computed checksum for non-deleted files
"""
file_path = file_service.path(path)
content, checksum = await file_service.read_file(file_path) if change_type != Change.deleted else (None, None)
return cls(path=file_path, change_type=change_type, checksum=checksum)
@dataclass
class SyncReport:
"""Report of file changes found compared to database state.
Attributes:
total: Total number of files in directory being synced
new: Files that exist on disk but not in database
@@ -48,19 +19,16 @@ class SyncReport:
moves: Files that have been moved from one location to another
checksums: Current checksums for files on disk
"""
total: int = 0
# We keep paths as strings in sets/dicts for easier serialization
new: Set[str] = field(default_factory=set)
modified: Set[str] = field(default_factory=set)
deleted: Set[str] = field(default_factory=set)
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
@property
def total_changes(self) -> int:
"""Total number of changes."""
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
@property
def syned_files(self) -> int:
"""Total number of files synced."""
return len(self.new) + len(self.modified) + len(self.moves)
+26 -5
View File
@@ -8,7 +8,6 @@ from datetime import datetime
from pathlib import Path
from typing import List, Optional
from rich import box
from rich.console import Console
from rich.live import Live
from rich.table import Table
@@ -84,7 +83,7 @@ class WatchService:
def generate_table(self) -> Table:
"""Generate status display table"""
table = Table(title="Basic Memory Sync Status")
table = Table()
# Add status row
table.add_column("Status", style="cyan")
@@ -131,13 +130,36 @@ class WatchService:
return table
async def run(self):
async def run(self, console_status: bool = False): # pragma: no cover
"""Watch for file changes and sync them"""
logger.info("Watching for sync changes")
self.state.running = True
self.state.start_time = datetime.now()
await self.write_status()
with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
if console_status:
with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
try:
async for changes in awatch(
self.config.home,
watch_filter=self.filter_changes,
debounce=self.config.sync_delay,
recursive=True,
):
# Process changes
await self.handle_changes(self.config.home)
# Update display
live.update(self.generate_table())
except Exception as e:
self.state.record_error(str(e))
await self.write_status()
raise
finally:
self.state.running = False
await self.write_status()
else:
try:
async for changes in awatch(
self.config.home,
@@ -148,7 +170,6 @@ class WatchService:
# Process changes
await self.handle_changes(self.config.home)
# Update display
live.update(self.generate_table())
except Exception as e:
self.state.record_error(str(e))
+42 -33
View File
@@ -1,38 +1,18 @@
"""Utility functions for basic-memory."""
import os
import re
import unicodedata
import sys
from pathlib import Path
from typing import Optional, Union
from loguru import logger
from unidecode import unidecode
def sanitize_name(name: str) -> str:
"""
Sanitize a name for filesystem use:
- Convert to lowercase
- Replace spaces/punctuation with underscores
- Remove emojis and other special characters
- Collapse multiple underscores
- Trim leading/trailing underscores
"""
# Normalize unicode to compose characters where possible
name = unicodedata.normalize("NFKD", name)
# Remove emojis and other special characters, keep only letters, numbers, spaces
name = "".join(c for c in name if c.isalnum() or c.isspace())
# Replace spaces with underscores
name = name.replace(" ", "_")
# Remove newline
name = name.replace("\n", "")
# Convert to lowercase
name = name.lower()
# Collapse multiple underscores and trim
name = re.sub(r"_+", "_", name).strip("_")
return name
from basic_memory.config import config
def generate_permalink(file_path: Path | str) -> str:
def generate_permalink(file_path: Union[Path, str]) -> str:
"""Generate a stable permalink from a file path.
Args:
@@ -50,8 +30,11 @@ def generate_permalink(file_path: Path | str) -> str:
>>> generate_permalink("design/unified_model_refactor.md")
'design/unified-model-refactor'
"""
# Convert Path to string if needed
path_str = str(file_path)
# Remove extension
base = os.path.splitext(file_path)[0]
base = os.path.splitext(path_str)[0]
# Transliterate unicode to ascii
ascii_text = unidecode(base)
@@ -63,16 +46,42 @@ def generate_permalink(file_path: Path | str) -> str:
lower_text = ascii_text.lower()
# replace underscores with hyphens
text_with_hyphens = lower_text.replace('_', '-')
text_with_hyphens = lower_text.replace("_", "-")
# Replace remaining invalid chars with hyphens
clean_text = re.sub(r'[^a-z0-9/\-]', '-', text_with_hyphens)
clean_text = re.sub(r"[^a-z0-9/\-]", "-", text_with_hyphens)
# Collapse multiple hyphens
clean_text = re.sub(r'-+', '-', clean_text)
clean_text = re.sub(r"-+", "-", clean_text)
# Clean each path segment
segments = clean_text.split('/')
clean_segments = [s.strip('-') for s in segments]
segments = clean_text.split("/")
clean_segments = [s.strip("-") for s in segments]
return '/'.join(clean_segments)
return "/".join(clean_segments)
def setup_logging(home_dir: Path = config.home, log_file: Optional[str] = None) -> None:
"""
Configure logging for the application.
"""
# Remove default handler and any existing handlers
logger.remove()
# Add file handler
if log_file:
log_path = home_dir / log_file
logger.add(
str(log_path), # loguru expects a string path
level=config.log_level,
rotation="100 MB",
retention="10 days",
backtrace=True,
diagnose=True,
enqueue=True,
colorize=False,
)
# Add stderr handler
logger.add(sys.stderr, level=config.log_level, backtrace=True, diagnose=True, colorize=True)
-458
View File
@@ -1,458 +0,0 @@
# Basic Memory Tasks
## Current Focus
### Observation Management
Implement update/remove functionality for observations with a focus on maintainability and consistency with our "
filesystem is source of truth" principle.
Options under consideration:
1. Bulk Update Approach
- Update all observations at once
- Pros:
- Simpler file operations
- No need to match on observation content
- Easier database synchronization
- Very consistent with "filesystem is source of truth"
- Cons:
- Less efficient - rewrites everything for small changes
- Potential concurrency implications
2. Tracked Observations Approach
- Use markdown comments for observation IDs
```markdown
# Entity Name
type: entity_type
## Observations
- <!-- obs-id: abc123 -->
This is an observation
```
- Pros:
- Can track individual observations
- Enables precise updates/deletes
- Cons:
- More complex markdown parsing
- IDs visible in markdown
3. Diff-based Approach
- Implement observation-aware diffing
- Track changes at observation level
- Pros:
- More efficient updates
- Preserves manual edits
- Cons:
- More complex implementation
- Need to handle merge conflicts
4. Position-based Management
- Track observations by their position/order
- Pros:
- No need for explicit IDs
- Clean markdown
- Cons:
- Fragile if order changes
- Hard to handle concurrent edits
## Completed
- [x] Extract file operations to fileio.py module
- [x] Update EntityService to use fileio functions
- [x] Initial ObservationService implementation
- [x] Basic test coverage
## Future Work
- [ ] Implement observation updates/removals (exploring options above)
- [ ] Proper session management for concurrent operations
- [ ] EntityService tests using new fileio module
- [ ] More sophisticated search functionality
- [ ] Handle markdown formatting edge cases
## TODO
### refactor input schema
1. Observations Format:
Old (JSON) way I tried first:
```python
"observations": ["First observation", "Second observation"] # Simple string array
```
New required format:
```python
"observations": [
{"content": "First observation"},
{"content": "Second observation"}
] # Array of objects with content field
```
2. Relations Format:
Old way:
```python
"relations": [
{"from": "EntityName", "to": "OtherEntity", "relationType": "relates_to"} # Using names
]
```
New format:
```python
"relations": [
{"fromId": "20241210-entity-id", "toId": "20241210-other-id", "relationType": "relates_to"} # Using IDs
]
```
My preferences:
1. For observations: The simple string array felt more intuitive for basic use, but I can see the benefits of the object
format:
- Allows for additional metadata (context, timestamps, etc.)
- More explicit about what each field means
- Consistent with how we'd want to store this in a database
2. For relations: Using IDs is technically better but requires an extra lookup step in my workflow:
- I have to first create the entities to get their IDs
- Then use those IDs to create relations
- Makes it harder to create entities and relations in a single step
Suggestions for making it more intuitive:
1. For observations: We could have a helper function that accepts either format:
```python
# Both would work:
create_entities([{
"name": "Entity",
"entityType": "type",
"observations": ["Simple string obs"] # Auto-converted to object format
}])
create_entities([{
"name": "Entity",
"entityType": "type",
"observations": [{"content": "Full object obs"}] # Native format
}])
```
2. For relations: Maybe allow a name-based helper function:
```python
# Instead of requiring IDs:
create_relations_by_name([{
"from": "EntityName",
"to": "OtherEntity",
"relationType": "relates_to"
}])
```
3. A combined creation function for when we want to create entities and their relations together:
```python
create_entity_with_relations({
"entity": {
"name": "NewEntity",
"entityType": "type",
"observations": ["Obs 1", "Obs 2"]
},
"relations": [{
"to": "ExistingEntity",
"relationType": "relates_to"
}]
})
```
The current schema is more "correct" from a software engineering perspective, but these helpers could make it more
natural to use while maintaining the rigorous underlying structure. What do you think about these suggestions? Would you
prefer to keep it strict for clarity, or add some convenience layers?
### entity file organization
- my problems
- wall of text is hard to scan visually
- files are not ordered or grouped granular enough. Only `<date>_<entity_name>.md`
- hard to tell when a new entity file is added
Possible fixes:
- use subdirectories?
## TASKS
1. **Core Functionality Improvements**
- [x] entity.description addition
- [x] subdirectories
- Paul needs this for markdown view
- [x] refine directory sprawl
- [ ] improve tool api
- [ ] Choose and implement observation update/removal strategy
- [ ] Handle relationship updates in files
- [ ] Complete full CRUD operations
- delete
- [ ] Improve search functionality (currently broken as we discovered)
### Suggested Sequence
1. **Schema Update First**
- Add `entity.description` field
- rename entity.references?
- This affects database, Pydantic models, and file format
- Good foundation for other changes
2. **File Organization**
- Add subdirectory support
- Affects:
- File path handling
- Entity loading/saving
- URI resolution
- Will make Paul's markdown viewing experience better
3. **Tool API Improvements**
- Cleaner input/output schemas
- More consistent patterns
- Better error handling
- This sets us up for implementing the remaining operations
4. **Core Operations**
- Implement delete operations
- Update/remove observations
- Relationship updates in files
- Building on the improved API
5. **Search Fix**
- Can properly tackle this after file organization
- Will benefit from improved schema
Would you like me to:
1. Start with the schema update for entity.description?
2. Plan out the subdirectory implementation?
3. Or focus on a different area?
I think the schema update would be a clean, contained change to start with, but I'm happy to tackle whichever part you think would be most valuable first.
2. **Robustness & Testing**
- Fix DI issues
- Learn from fastmcp patterns
- Markdown service
- markdown.py
-python-frontmatter
- Complete test coverage
- Expand testing across services
- 100% coverate
- Improve error handling and logging
- Add comprehensive type hints
3. **Architecture Improvements**
- Handle concurrent file operations safely
- Implement proper session management
- Balance file operations and DB sync
- Handle markdown formatting edge cases
4. **Documentation & Infrastructure**
- Document core modules
- Implement proper backup strategy
- Add file change versioning
- Improve CLI interface
## Ideas
- need update tool
### 2-way sync
- Enable updates to the markdown files to be able to be seen by AI
- possible via tool sync
- filesystem notifications via agent?
- Claude can use `file_write` tool to edit Entity files also
### Projects
- support multiple projects
- figure out flow
- load project at startup?
- switch project during chat?
### References
- better support for ref:// references
- should we call them memory://<project>/<entity>
- use Prompt to invoke reference uri
### Entity File format
- Consider using wiki link format:
[[20241210-ai_human_collaboration_model]] improves
instead of
[20241210-basic-memory] validates_design_of
- use frontmatter for file header
### Code
- cleanup server.py
- use markdown.py?
- fastmcp?
- Entity.description
### Other Features
- Search
- webui
- python notebook?
- vector eb extension for sqlite
#### Obsidian
- use as file viewer editor
- create knowledge graph?
#### Save chats
I'll sketch out a potential schema for chat sessions in our system:
```
{
`entities`: [
{
`name`: `Chat_Entity_Design`,
`entityType`: `design_proposal`,
`observations`: [
{
`content`: `Entity Schema:
type: chat_session
date: ISO timestamp
participants: [human, ai]
topic: string
context_refs: [memory://<project>/<entity_id>]
key_decisions: [string]
summary: string`
},
{
`content`: `File Format (frontmatter):
---
type: chat_session
date: 2024-12-10T10:30:00Z
participants:
- human: paul
- ai: claude
topic: Chat Session Storage Design
context_refs:
- memory://basic-memory/20241210-chat-entity-design
key_decisions:
- Store chats as first-class entities
- Use memory:// URI scheme
summary: Designed schema for chat storage
---`
},
{
`content`: `Example Relations:
- chat_session references design_proposal
- chat_session implements feature
- chat_session discusses concept
- chat_session continues_from previous_chat`
},
{
`content`: `Auto-generation approach:
1. At end of chat, Claude summarizes key points
2. Creates chat entity with metadata
3. Adds relations to referenced entities
4. Updates any entities modified during chat`
},
{
`content`: `Integration points:
- memory:// URI scheme for references
- Relations to track context chain
- File format matches entity schema
- Automatic metadata generation`
}
]
}
]
}
```
```
{
`entities`: [
{
`name`: `Chat_Session_Workflow`,
`entityType`: `process`,
`observations`: [
{
`content`: `Start of chat:
- Load any referenced context via memory:// URIs
- Initialize new chat session metadata
- Track entities being discussed/modified`
},
{
`content`: `During chat:
- Update references as new entities are mentioned
- Track key decisions and changes
- Note any new relations discovered`
},
{
`content`: `End of chat:
- Generate summary and key points
- Create chat session entity
- Add relations to referenced entities
- Update any modified entities`
},
{
`content`: `Future chat reference:
- Can reference previous chat via memory:// URI
- Loads relevant context and decisions
- Creates continues_from relation automatically`
}
]
}
]
}
```
Key aspects of this design:
1. Chat Session as Entity
- Clear metadata (participants, date, topic)
- Explicit references to other entities via memory:// URIs
- Captures key decisions and summary
- Uses frontmatter for clean file format
2. Rich Relations
- Can track chat-to-chat continuity
- Links to entities discussed/modified
- Captures purpose (discusses, implements, etc.)
- Maintains context chain
3. Automated Workflow
- I could generate metadata during chat
- Create summary and relations at end
- Update any modified entities
- Makes context loading explicit
4. Integration
- memory:// URI scheme fits naturally
- File format matches our entity pattern
- Two-way links between chats and entities
- Clear context persistence model
The nice thing about this design is that it makes chat sessions "first-class citizens" in our knowledge graph while
keeping everything in human-readable markdown files.
Would you like me to:
1. Sketch out what this chat would look like as an entity?
2. Detail the MCP tool changes needed?
3. Mock up the file format more precisely?
4. Something else?
+18 -6
View File
@@ -77,8 +77,6 @@ async def test_create_entity_observations_relations(client: AsyncClient, file_se
assert entity.relations[0].from_id == "test/test-entity"
assert entity.relations[0].to_id is None
# TODO Relation.to_id should be name from link
# Verify file has new content but preserved metadata
file_path = file_service.get_entity_path(entity)
file_content, _ = await file_service.read_file(file_path)
@@ -111,8 +109,12 @@ async def test_get_entity(client: AsyncClient):
async def test_get_entities(client: AsyncClient):
"""Should open multiple entities by path IDs."""
# Create a few entities with different names
await client.post("/knowledge/entities", json={"title": "AlphaTest", "folder": "", "entity_type": "test"})
await client.post("/knowledge/entities", json={"title": "BetaTest", "folder": "", "entity_type": "test"})
await client.post(
"/knowledge/entities", json={"title": "AlphaTest", "folder": "", "entity_type": "test"}
)
await client.post(
"/knowledge/entities", json={"title": "BetaTest", "folder": "", "entity_type": "test"}
)
# Open nodes by path IDs
response = await client.get(
@@ -397,7 +399,12 @@ async def test_update_entity_type_conversion(client: AsyncClient):
async def test_update_entity_metadata(client: AsyncClient):
"""Test updating entity metadata."""
# Create entity
data = {"title": "test", "folder": "", "entity_type": "test", "entity_metadata": {"status": "draft"}}
data = {
"title": "test",
"folder": "",
"entity_type": "test",
"entity_metadata": {"status": "draft"},
}
response = await client.post("/knowledge/entities", json=data)
entity_response = response.json()
@@ -450,7 +457,12 @@ async def test_update_entity_incorrect_permalink(client: AsyncClient):
async def test_update_entity_search_index(client: AsyncClient):
"""Test search index is updated after entity changes."""
# Create entity
data = {"title": "test", "folder": "", "entity_type": "test", "content": "Initial searchable content"}
data = {
"title": "test",
"folder": "",
"entity_type": "test",
"content": "Initial searchable content",
}
response = await client.post("/knowledge/entities", json=data)
entity_response = response.json()
+1 -10
View File
@@ -23,7 +23,7 @@ async def test_get_memory_context(client, test_graph):
assert context.metadata.depth == 1 # default depth
# assert context.metadata["timeframe"] == "7d" # default timeframe
assert isinstance(context.metadata.generated_at, datetime)
assert context.metadata.total_results == 2
assert context.metadata.total_results == 3
@pytest.mark.asyncio
@@ -69,15 +69,6 @@ async def test_get_memory_context_timeframe(client, test_graph):
assert len(older.related_results) >= len(recent.related_results)
@pytest.mark.asyncio
async def test_get_related_context_filters(client, test_graph):
"""Test filtering related content by relation type."""
response = await client.get("/memory/related/test/root")
assert response.status_code == 200
context = GraphContext(**response.json())
@pytest.mark.asyncio
async def test_not_found(client):
"""Test handling of non-existent paths."""
+1
View File
@@ -35,6 +35,7 @@ async def test_get_resource_content(client, test_config, entity_repository):
assert response.text == content
@pytest.mark.asyncio
async def test_get_resource_by_title(client, test_config, entity_repository):
"""Test getting content by permalink."""
# Create a test file
+8 -8
View File
@@ -11,7 +11,6 @@ from basic_memory.schemas import Entity as EntitySchema
from basic_memory.schemas.search import SearchItemType, SearchResponse
@pytest_asyncio.fixture
async def indexed_entity(init_search_index, full_entity, search_service):
"""Create an entity and index it."""
@@ -22,17 +21,17 @@ async def indexed_entity(init_search_index, full_entity, search_service):
@pytest.mark.asyncio
async def test_search_basic(client, indexed_entity):
"""Test basic text search."""
response = await client.post("/search/", json={"text": "searchable"})
response = await client.post("/search/", json={"text": "search"})
assert response.status_code == 200
search_results = SearchResponse.model_validate(response.json())
assert len(search_results.results) == 3
found = False
for r in search_results.results:
if r.type == SearchItemType.ENTITY.value:
if r.type == SearchItemType.ENTITY.value:
assert r.permalink == indexed_entity.permalink
found = True
assert found, "Expected to find indexed entity in results"
@@ -45,13 +44,15 @@ async def test_search_with_type_filter(client, indexed_entity):
)
assert response.status_code == 200
search_results = SearchResponse.model_validate(response.json())
assert len(search_results.results) > 0
# Should not find with wrong type
# Should find with relation type
response = await client.post(
"/search/", json={"text": "test", "types": [SearchItemType.RELATION.value]}
)
assert response.status_code == 200
search_results = SearchResponse.model_validate(response.json())
assert len(search_results.results) == 2
@pytest.mark.asyncio
@@ -90,6 +91,7 @@ async def test_search_with_date_filter(client, indexed_entity):
search_results = SearchResponse.model_validate(response.json())
assert len(search_results.results) == 0
@pytest.mark.skip("search scoring is not implemented yet")
@pytest.mark.asyncio
async def test_search_scoring(client, indexed_entity):
@@ -152,8 +154,6 @@ async def test_reindex(client, search_service, entity_service, session_maker):
search_response = await client.post("/search/", json={"text": "test"})
search_results = SearchResponse.model_validate(search_response.json())
assert len(search_results.results) == 1
@pytest.mark.asyncio
+105 -146
View File
@@ -1,175 +1,134 @@
"""Test import-json command functionality."""
"""Tests for import_memory_json command."""
import json
from io import StringIO
from pathlib import Path
import pytest
from rich.console import Console
from typer.testing import CliRunner
from basic_memory.cli.commands.import_memory_json import process_memory_json
from basic_memory.cli.app import app
from basic_memory.cli.commands import import_memory_json
from basic_memory.markdown import EntityParser, MarkdownProcessor
@pytest.fixture
def console():
"""Create test console that captures output."""
output = StringIO()
return Console(file=output), output
# Set up CLI runner
runner = CliRunner()
@pytest.fixture
def sample_memory_json(tmp_path) -> Path:
"""Create a sample memory.json file with test data."""
json_path = tmp_path / "memory.json"
# Create test data modeling the real format
test_data = [
def sample_entities():
"""Sample entities for testing."""
return [
{
"type": "entity",
"name": "Basic_Memory",
"entityType": "software_system",
"observations": [
"A core component of Basic Machines",
"Local-first knowledge management system",
"Combines filesystem persistence with graph-based knowledge representation"
]
},
{
"type": "entity",
"name": "Basic_Machines",
"entityType": "project",
"observations": [
"Local-first knowledge management system",
"Focuses on enhancing human agency and understanding",
"Current focus includes basic-memory system"
]
"name": "test_entity",
"entityType": "test",
"observations": ["Test observation 1", "Test observation 2"],
},
{
"type": "relation",
"from": "Basic_Memory",
"to": "Basic_Machines",
"relationType": "is_component_of"
}
"from": "test_entity",
"to": "related_entity",
"relationType": "test_relation",
},
]
# Write each item as a JSON line
with open(json_path, 'w') as f:
for item in test_data:
f.write(json.dumps(item) + '\n')
return json_path
@pytest.fixture
def sample_json_file(tmp_path, sample_entities):
"""Create a sample memory.json file."""
json_file = tmp_path / "memory.json"
with open(json_file, "w") as f:
for entity in sample_entities:
f.write(json.dumps(entity) + "\n")
return json_file
@pytest.mark.asyncio
async def test_process_memory_json(
sample_memory_json: Path,
markdown_processor: MarkdownProcessor,
test_config,
):
"""Test importing from memory.json format."""
# Process the import
results = await process_memory_json(sample_memory_json, test_config.home, markdown_processor)
# Check results
assert results["entities"] == 2
async def test_process_memory_json(tmp_path, sample_json_file):
"""Test importing entities from JSON."""
entity_parser = EntityParser(tmp_path)
processor = MarkdownProcessor(entity_parser)
results = await import_memory_json.process_memory_json(sample_json_file, tmp_path, processor)
assert results["entities"] == 1
assert results["relations"] == 1
# Verify Basic_Memory entity file was created correctly
basic_memory_path = test_config.home / "software_system/Basic_Memory.md"
entity = await markdown_processor.read_file(basic_memory_path)
assert entity.frontmatter.title == "Basic_Memory"
assert entity.frontmatter.type == "software_system"
assert len(entity.observations) == 3
assert len(entity.relations) == 1 # Should have the outgoing relation
assert entity.relations[0].type == "is_component_of"
assert entity.relations[0].target == "Basic_Machines"
# Verify Basic_Machines entity file
basic_machines_path = test_config.home / "project/Basic_Machines.md"
entity = await markdown_processor.read_file(basic_machines_path)
assert entity.frontmatter.title == "Basic_Machines"
assert entity.frontmatter.type == "project"
assert len(entity.observations) == 3
assert len(entity.relations) == 0 # No outgoing relations
# Check file was created
entity_file = tmp_path / "test/test_entity.md"
assert entity_file.exists()
content = entity_file.read_text()
assert "Test observation 1" in content
assert "Test observation 2" in content
assert "test_relation [[related_entity]]" in content
@pytest.mark.asyncio
async def test_process_memory_json_empty_observations(
tmp_path: Path,
markdown_processor: MarkdownProcessor,
test_config,
):
"""Test handling entities with no observations."""
# Create test data
json_path = tmp_path / "memory.json"
test_data = [
{
"type": "entity",
"name": "Empty_Entity",
"entityType": "test",
"observations": [] # Empty observations
}
]
with open(json_path, 'w') as f:
for item in test_data:
f.write(json.dumps(item) + '\n')
# Process import
results = await process_memory_json(json_path, test_config.home, markdown_processor)
# Check results
assert results["entities"] == 1
assert results["relations"] == 0
# Verify file was created
entity_path = test_config.home / "test/Empty_Entity.md"
entity = await markdown_processor.read_file(entity_path)
assert entity.frontmatter.title == "Empty_Entity"
assert entity.observations == []
assert entity.relations == []
async def test_get_markdown_processor(tmp_path, monkeypatch):
"""Test getting markdown processor."""
monkeypatch.setenv("HOME", str(tmp_path))
processor = await import_memory_json.get_markdown_processor()
assert isinstance(processor, MarkdownProcessor)
@pytest.mark.asyncio
async def test_process_memory_json_special_characters(
tmp_path: Path,
markdown_processor: MarkdownProcessor,
test_config,
):
"""Test handling entities with special characters in text."""
# Create test data
json_path = tmp_path / "memory.json"
test_data = [
def test_import_json_command_file_not_found(tmp_path):
"""Test error handling for nonexistent file."""
nonexistent = tmp_path / "nonexistent.json"
result = runner.invoke(app, ["import-json", str(nonexistent)])
assert result.exit_code == 1
assert "File not found" in result.output
def test_import_json_command_success(tmp_path, sample_json_file, monkeypatch):
"""Test successful JSON import via command."""
# Set up test environment
monkeypatch.setenv("HOME", str(tmp_path))
# Run import
result = runner.invoke(app, ["import-json", str(sample_json_file)])
assert result.exit_code == 0
assert "Import complete" in result.output
assert "Created 1 entities" in result.output
assert "Added 1 relations" in result.output
def test_import_json_command_invalid_json(tmp_path):
"""Test error handling for invalid JSON."""
# Create invalid JSON file
invalid_file = tmp_path / "invalid.json"
invalid_file.write_text("not json")
result = runner.invoke(app, ["import-json", str(invalid_file)])
assert result.exit_code == 1
assert "Error during import" in result.output
def test_import_json_command_handle_old_format(tmp_path):
"""Test handling old format JSON with from_id/to_id."""
# Create JSON with old format
old_format = [
{
"type": "entity",
"name": "Special_Entity",
"name": "test_entity",
"entityType": "test",
"observations": [
"Contains *markdown* formatting",
"Has #hashtags and @mentions",
"Uses [square brackets] and {curly braces}"
]
}
"observations": ["Test observation"],
},
{
"type": "relation",
"from_id": "test_entity",
"to_id": "other_entity",
"relation_type": "test_relation",
},
]
with open(json_path, 'w') as f:
for item in test_data:
f.write(json.dumps(item) + '\n')
# Process import
results = await process_memory_json(json_path, test_config.home, markdown_processor)
assert results["entities"] == 1
# Verify file was created and content preserved
entity_path = test_config.home / "test/Special_Entity.md"
entity = await markdown_processor.read_file(entity_path)
assert len(entity.observations) == 3
assert entity.observations[0].content == "Contains *markdown* formatting"
assert entity.observations[1].content == "Has #hashtags and @mentions"
assert entity.observations[2].content == "Uses [square brackets] and {curly braces}"
json_file = tmp_path / "old_format.json"
with open(json_file, "w") as f:
for item in old_format:
f.write(json.dumps(item) + "\n")
# Set up test environment
monkeypatch = pytest.MonkeyPatch()
monkeypatch.setenv("HOME", str(tmp_path))
# Run import
result = runner.invoke(app, ["import-json", str(json_file)])
assert result.exit_code == 0
assert "Import complete" in result.output
+160
View File
@@ -0,0 +1,160 @@
"""Tests for CLI status command."""
import pytest
import pytest_asyncio
from typer.testing import CliRunner
from basic_memory.cli.app import app
from basic_memory.cli.commands.status import (
add_files_to_tree,
build_directory_summary,
group_changes_by_directory,
run_status,
display_changes,
)
from basic_memory.sync.utils import SyncReport
from basic_memory.sync import FileChangeScanner
from basic_memory.repository import EntityRepository
# Set up CLI runner
runner = CliRunner()
@pytest_asyncio.fixture
async def file_change_scanner(session_maker):
"""Create FileChangeScanner instance with test database."""
entity_repository = EntityRepository(session_maker)
scanner = FileChangeScanner(entity_repository)
return scanner
@pytest.mark.asyncio
async def test_run_status_no_changes(file_change_scanner, tmp_path, monkeypatch):
"""Test status command with no changes."""
# Set up test environment
monkeypatch.setenv("HOME", str(tmp_path))
knowledge_dir = tmp_path / "knowledge"
knowledge_dir.mkdir()
# Run status check
await run_status(file_change_scanner, verbose=False)
@pytest.mark.asyncio
async def test_run_status_with_changes(file_change_scanner, tmp_path, monkeypatch):
"""Test status command with actual file changes."""
# Set up test environment
monkeypatch.setenv("HOME", str(tmp_path))
knowledge_dir = tmp_path / "knowledge"
knowledge_dir.mkdir()
# Create test files
test_file = knowledge_dir / "test.md"
test_file.write_text("# Test\nSome content")
# Run status check - should detect new file
await run_status(file_change_scanner, verbose=True)
@pytest.mark.asyncio
async def test_status_command_error(tmp_path, monkeypatch):
"""Test CLI status command error handling."""
# Set up invalid environment
nonexistent = tmp_path / "nonexistent"
monkeypatch.setenv("HOME", str(nonexistent))
monkeypatch.setenv("DATABASE_PATH", str(nonexistent / "nonexistent.db"))
# Should exit with code 1 when error occurs
result = runner.invoke(app, ["status", "--verbose"])
assert result.exit_code == 1
def test_display_changes_no_changes():
"""Test displaying no changes."""
changes = SyncReport(set(), set(), set(), {}, {})
display_changes("Test", changes, verbose=True)
display_changes("Test", changes, verbose=False)
def test_display_changes_with_changes():
"""Test displaying various changes."""
changes = SyncReport(
new={"dir1/new.md"},
modified={"dir1/mod.md"},
deleted={"dir2/del.md"},
moves={"old.md": "new.md"},
checksums={"dir1/new.md": "abcd1234"},
)
display_changes("Test", changes, verbose=True)
display_changes("Test", changes, verbose=False)
def test_build_directory_summary():
"""Test building directory change summary."""
counts = {
"new": 2,
"modified": 1,
"moved": 1,
"deleted": 1,
}
summary = build_directory_summary(counts)
assert "+2" in summary
assert "~1" in summary
assert "↔1" in summary
assert "-1" in summary
def test_build_directory_summary_empty():
"""Test summary with no changes."""
counts = {
"new": 0,
"modified": 0,
"moved": 0,
"deleted": 0,
}
summary = build_directory_summary(counts)
assert summary == ""
def test_group_changes_by_directory():
"""Test grouping changes by directory."""
changes = SyncReport(
new={"dir1/new.md", "dir2/new2.md"},
modified={"dir1/mod.md"},
deleted={"dir2/del.md"},
moves={"dir1/old.md": "dir2/new.md"},
checksums={},
)
grouped = group_changes_by_directory(changes)
assert grouped["dir1"]["new"] == 1
assert grouped["dir1"]["modified"] == 1
assert grouped["dir1"]["moved"] == 1
assert grouped["dir2"]["new"] == 1
assert grouped["dir2"]["deleted"] == 1
assert grouped["dir2"]["moved"] == 1
def test_add_files_to_tree():
"""Test adding files to tree visualization."""
from rich.tree import Tree
# Test with various path patterns
paths = {
"dir1/file1.md", # Normal nested file
"dir1/file2.md", # Another in same dir
"dir2/subdir/file3.md", # Deeper nesting
"root.md", # Root level file
}
# Test without checksums
tree = Tree("Test")
add_files_to_tree(tree, paths, "green")
# Test with checksums
checksums = {"dir1/file1.md": "abcd1234", "dir1/file2.md": "efgh5678"}
tree = Tree("Test with checksums")
add_files_to_tree(tree, paths, "green", checksums)
+109
View File
@@ -0,0 +1,109 @@
"""Tests for CLI sync command."""
import asyncio
import pytest
from typer.testing import CliRunner
from basic_memory.cli.app import app
from basic_memory.cli.commands.sync import (
display_sync_summary,
display_detailed_sync_results,
run_sync,
group_issues_by_directory,
ValidationIssue,
)
from basic_memory.config import config
from basic_memory.sync.utils import SyncReport
# Set up CLI runner
runner = CliRunner()
def test_group_issues_by_directory():
"""Test grouping validation issues by directory."""
issues = [
ValidationIssue("dir1/file1.md", "error1"),
ValidationIssue("dir1/file2.md", "error2"),
ValidationIssue("dir2/file3.md", "error3"),
]
grouped = group_issues_by_directory(issues)
assert len(grouped["dir1"]) == 2
assert len(grouped["dir2"]) == 1
assert grouped["dir1"][0].error == "error1"
assert grouped["dir2"][0].error == "error3"
def test_display_sync_summary_no_changes():
"""Test displaying sync summary with no changes."""
changes = SyncReport(set(), set(), set(), {}, {})
display_sync_summary(changes)
def test_display_sync_summary_with_changes():
"""Test displaying sync summary with various changes."""
changes = SyncReport(
new={"new.md"},
modified={"mod.md"},
deleted={"del.md"},
moves={"old.md": "new.md"},
checksums={"new.md": "abcd1234"},
)
display_sync_summary(changes)
def test_display_detailed_sync_results_no_changes():
"""Test displaying detailed results with no changes."""
changes = SyncReport(set(), set(), set(), {}, {})
display_detailed_sync_results(changes)
def test_display_detailed_sync_results_with_changes():
"""Test displaying detailed results with various changes."""
changes = SyncReport(
new={"new.md"},
modified={"mod.md"},
deleted={"del.md"},
moves={"old.md": "new.md"},
checksums={"new.md": "abcd1234", "mod.md": "efgh5678"},
)
display_detailed_sync_results(changes)
@pytest.mark.asyncio
async def test_run_sync_basic(sync_service, test_config):
"""Test basic sync operation."""
# Set up test environment
config.home = test_config.home
# Create test files
test_file = test_config.home / "test.md"
test_file.write_text("""---
title: Test
---
# Test
Some content""")
# Run sync - should detect new file
await run_sync(verbose=True)
@pytest.mark.asyncio
async def test_run_sync_watch_mode(sync_service, test_config):
"""Test sync with watch mode."""
# Set up test environment
config.home = test_config.home
# Start sync in watch mode but cancel after a short time
with pytest.raises(asyncio.CancelledError):
task = asyncio.create_task(run_sync(watch=True))
await asyncio.sleep(0.1) # Let it start
task.cancel()
await task
def test_sync_command():
"""Test the sync command."""
result = runner.invoke(app, ["sync", "--verbose"])
assert result.exit_code == 0
+113 -161
View File
@@ -1,5 +1,6 @@
"""Common test fixtures."""
from textwrap import dedent
from typing import AsyncGenerator
from datetime import datetime, timezone
@@ -11,18 +12,18 @@ from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, async_sessionmaker
from basic_memory import db
from basic_memory.config import ProjectConfig
from basic_memory.db import DatabaseType, init_db
from basic_memory.db import DatabaseType
from basic_memory.markdown import EntityParser
from basic_memory.markdown.markdown_processor import MarkdownProcessor
from basic_memory.models import Base
from basic_memory.models.knowledge import Entity, Observation, ObservationCategory, Relation
from basic_memory.models.knowledge import Entity
from basic_memory.repository.entity_repository import EntityRepository
from basic_memory.repository.observation_repository import ObservationRepository
from basic_memory.repository.relation_repository import RelationRepository
from basic_memory.repository.search_repository import SearchRepository
from basic_memory.schemas.base import Entity as EntitySchema
from basic_memory.services import (
EntityService,
DatabaseService,
EntityService,
)
from basic_memory.services.file_service import FileService
from basic_memory.services.link_resolver import LinkResolver
@@ -146,7 +147,6 @@ def file_change_scanner(entity_repository) -> FileChangeScanner:
return FileChangeScanner(entity_repository)
@pytest_asyncio.fixture
async def sync_service(
file_change_scanner: FileChangeScanner,
@@ -206,180 +206,136 @@ async def sample_entity(entity_repository: EntityRepository) -> Entity:
@pytest_asyncio.fixture
async def full_entity(sample_entity, entity_repository, file_service) -> Entity:
async def full_entity(sample_entity, entity_repository, file_service, entity_service) -> Entity:
"""Create a search test entity."""
search_entity = await entity_repository.create(
{
"title": "Searchable Entity",
"entity_type": "test",
"permalink": "test/search-entity",
"file_path": "test/search_entity.md",
"content_type": "text/markdown",
"created_at": datetime.now(timezone.utc),
"updated_at": datetime.now(timezone.utc),
}
# Create test entity
entity, created = await entity_service.create_or_update_entity(
EntitySchema(
title="Search_Entity",
folder="test",
entity_type="test",
content=dedent("""
## Observations
- [tech] Tech note
- [design] Design note
## Relations
- out1 [[Test Entity]]
- out2 [[Test Entity]]
"""),
)
)
observations = [
Observation(
content="Tech note",
category=ObservationCategory.TECH,
),
Observation(
content="Design note",
category=ObservationCategory.DESIGN,
),
]
relations = [
Relation(
from_id=search_entity.id,
to_id=sample_entity.id,
to_name=sample_entity.title,
relation_type="out1",
),
Relation(
from_id=search_entity.id,
to_id=sample_entity.id,
to_name=sample_entity.title,
relation_type="out2",
),
]
search_entity.observations = observations
search_entity.outgoing_relations = relations
full_entity = await entity_repository.add(search_entity)
# write file
await file_service.write_entity_file(full_entity)
return full_entity
return entity
@pytest_asyncio.fixture
async def test_graph(
entity_repository, relation_repository, observation_repository, search_service, file_service
entity_repository,
relation_repository,
observation_repository,
search_service,
file_service,
entity_service,
):
"""Create a test knowledge graph with entities, relations and observations."""
# Create some test entities
entities = [
Entity(
title="Root Entity",
entity_type="test",
permalink="test/root",
file_path="test/root.md",
content_type="text/markdown",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
),
Entity(
title="Connected Entity 1",
entity_type="test",
permalink="test/connected1",
file_path="test/connected1.md",
content_type="text/markdown",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
),
Entity(
title="Connected Entity 2",
entity_type="test",
permalink="test/connected2",
file_path="test/connected2.md",
content_type="text/markdown",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
),
Entity(
title="Deep Entity",
entity_type="deep",
permalink="test/deep",
file_path="test/deep.md",
content_type="text/markdown",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
),
Entity(
# Create some test entities in reverse order so they will be linked
deeper, _ = await entity_service.create_or_update_entity(
EntitySchema(
title="Deeper Entity",
entity_type="deeper",
permalink="test/deeper",
file_path="test/deeper.md",
content_type="text/markdown",
created_at=datetime.now(timezone.utc),
updated_at=datetime.now(timezone.utc),
),
]
entities = await entity_repository.add_all(entities)
root, conn1, conn2, deep, deeper = entities
# Add some observations
root.observations = [
Observation(
content="Root note 1",
category=ObservationCategory.NOTE,
),
Observation(
content="Root tech note",
category=ObservationCategory.TECH,
),
]
conn1.observations = [
Observation(
content="Connected 1 note",
category=ObservationCategory.NOTE,
folder="test",
content=dedent("""
# Deeper Entity
"""),
)
]
await observation_repository.add_all(root.observations)
await observation_repository.add_all(conn1.observations)
)
# Add relations
relations = [
# Direct connections to root
Relation(
from_id=root.id,
to_id=conn1.id,
to_name=conn1.title,
relation_type="connects_to",
),
Relation(
from_id=conn1.id,
to_id=conn2.id,
to_name=conn2.title,
relation_type="connected_to",
),
# Deep connection
Relation(
from_id=conn2.id,
to_id=deep.id,
to_name=deep.title,
relation_type="deep_connection",
),
# Deeper connection
Relation(
from_id=deep.id,
to_id=deeper.id,
to_name=deeper.title,
relation_type="deeper_connection",
),
]
deep, _ = await entity_service.create_or_update_entity(
EntitySchema(
title="Deep Entity",
entity_type="deep",
folder="test",
content=dedent("""
# Deep Entity
- deeper_connection [[Deeper Entity]]
"""),
)
)
# Save relations
related_entities = await relation_repository.add_all(relations)
connected_2, _ = await entity_service.create_or_update_entity(
EntitySchema(
title="Connected Entity 2",
entity_type="test",
folder="test",
content=dedent("""
# Connected Entity 2
- deep_connection [[Deep Entity]]
"""),
)
)
connected_1, _ = await entity_service.create_or_update_entity(
EntitySchema(
title="Connected Entity 1",
entity_type="test",
folder="test",
content=dedent("""
# Connected Entity 1
- [note] Connected 1 note
- connected_to [[Connected Entity 2]]
"""),
)
)
root, _ = await entity_service.create_or_update_entity(
EntitySchema(
title="Root",
entity_type="test",
folder="test",
content=dedent("""
# Root Entity
- [note] Root note 1
- [tech] Root tech note
- connects_to [[Connected Entity 1]]
"""),
)
)
# get latest
entities = await entity_repository.find_all()
# make sure we have files for entities
for entity in entities:
await file_service.write_entity_file(entity)
relations = await relation_repository.find_all()
# Index everything for search
for entity in entities:
await search_service.index_entity(entity)
# search_content = await entity_repository.execute_query(text("select * from search_index"),
# use_query_options=False)
# for row in search_content:
# print(row)
# print("relation:")
# search_content = await entity_repository.execute_query(text("select * from search_index where type = 'relation'"),
# use_query_options=False)
# for row in search_content:
# print(row)
#
# # In test_graph fixture after creating everything:
# print("Entities:")
# entities = await entity_repository.find_all()
# for e in entities:
# print(f"- {e.title} (id={e.id})")
#
# print("\nRelations:")
# relations = await relation_repository.find_all()
# for r in relations:
# print(f"- {r.from_id} -> {r.to_id} ({r.relation_type})")
return {
"root": entities[0],
"connected1": conn1,
"connected2": conn2,
"root": root,
"connected1": connected_1,
"connected2": connected_2,
"deep": deep,
"observations": [e.observations for e in entities],
"relations": relations,
@@ -388,8 +344,4 @@ async def test_graph(
@pytest_asyncio.fixture
def watch_service(sync_service, file_service, test_config):
return WatchService(
sync_service=sync_service,
file_service=file_service,
config=test_config
)
return WatchService(sync_service=sync_service, file_service=file_service, config=test_config)
+5 -3
View File
@@ -1,17 +1,19 @@
"""Test file for experimenting with edit_file."""
def function_one():
"""First test function."""
print("Hello from function one")
# Some code here
x = 1 + 2
return x
def function_two():
"""Second test function."""
print("Hello from function two")
# Some more code
y = 3 * 4
return y
return y
+67 -14
View File
@@ -1,12 +1,13 @@
"""Tests for entity markdown parsing."""
import os
from datetime import datetime, timedelta, UTC
from datetime import datetime
from pathlib import Path
from textwrap import dedent
import pytest
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Relation
from basic_memory.markdown.entity_parser import parse
@pytest.fixture
@@ -76,8 +77,7 @@ async def test_parse_complete_file(test_config, entity_parser, valid_entity_cont
in entity.relations
), "missing [[OAuth Implementation]]"
assert (
Relation(type="uses", target="Redis Cache", context="Token caching")
in entity.relations
Relation(type="uses", target="Redis Cache", context="Token caching") in entity.relations
), "missing [[Redis Cache]]"
assert (
Relation(type="specified_by", target="Auth API Spec", context="OpenAPI spec")
@@ -85,9 +85,9 @@ async def test_parse_complete_file(test_config, entity_parser, valid_entity_cont
), "missing [[Auth API Spec]]"
# inline links in content
assert (
Relation(type="links to", target="Random Link", context=None) in entity.relations
), "missing [[Random Link]]"
assert Relation(type="links to", target="Random Link", context=None) in entity.relations, (
"missing [[Random Link]]"
)
assert (
Relation(type="links to", target="Random Link with Title|Titled Link", context=None)
in entity.relations
@@ -121,7 +121,7 @@ async def test_parse_minimal_file(test_config, entity_parser):
assert entity.frontmatter.permalink is None
assert len(entity.observations) == 1
assert len(entity.relations) == 1
assert entity.created is not None
assert entity.modified is not None
@@ -141,6 +141,7 @@ async def test_error_handling(test_config, entity_parser):
with pytest.raises(UnicodeDecodeError):
await entity_parser.parse_file(test_file)
@pytest.mark.asyncio
async def test_parse_file_without_section_headers(test_config, entity_parser):
"""Test parsing a minimal valid entity file."""
@@ -169,17 +170,69 @@ async def test_parse_file_without_section_headers(test_config, entity_parser):
assert entity.frontmatter.type == "component"
assert entity.frontmatter.permalink == "minimal_entity"
assert "some text\nsome [[Random Link]]" in entity.content
assert "some text\nsome [[Random Link]]" in entity.content
assert len(entity.observations) == 1
assert entity.observations[0].category == "note"
assert entity.observations[0].content == "Basic observation #test"
assert entity.observations[0].tags == ["test"]
assert len(entity.relations) == 2
assert entity.relations[0].type == "links to"
assert entity.relations[0].target == "Random Link"
assert entity.relations[1].type == "references"
assert entity.relations[1].target == "Other Entity"
assert entity.relations[1].target == "Other Entity"
def test_parse_date_formats(entity_parser):
"""Test date parsing functionality."""
# Valid formats
assert entity_parser.parse_date("2024-01-15") is not None
assert entity_parser.parse_date("Jan 15, 2024") is not None
assert entity_parser.parse_date("2024-01-15 10:00 AM") is not None
assert entity_parser.parse_date(datetime.now()) is not None
# Invalid formats
assert entity_parser.parse_date(None) is None
assert entity_parser.parse_date(123) is None # Non-string/datetime
assert entity_parser.parse_date("not a date") is None # Unparseable string
assert entity_parser.parse_date("") is None # Empty string
# Test dateparser error handling
assert entity_parser.parse_date("25:00:00") is None # Invalid time
def test_parse_empty_content():
"""Test parsing empty or minimal content."""
result = parse("")
assert result.content == ""
assert len(result.observations) == 0
assert len(result.relations) == 0
result = parse("# Just a title")
assert result.content == "# Just a title"
assert len(result.observations) == 0
assert len(result.relations) == 0
# @pytest.mark.asyncio
# async def test_parse_file_invalid_yaml(test_config, entity_parser):
# """Test parsing file with invalid YAML frontmatter."""
# content = dedent("""
# ---
# invalid: [yaml: ]syntax]
# ---
#
# # Invalid YAML Frontmatter
# """)
#
# test_file = test_config.home / "invalid_yaml.md"
# test_file.write_text(content)
#
# # Should handle invalid YAML gracefully
# entity = await entity_parser.parse_file(test_file)
# assert entity.frontmatter.title == "invalid_yaml.md"
# assert entity.frontmatter.type == "note"
# assert entity.content.strip() == "# Invalid YAML Frontmatter"
+139 -137
View File
@@ -1,163 +1,165 @@
"""Tests for markdown-it plugins."""
"""Tests for markdown plugins."""
import pytest
from textwrap import dedent
from markdown_it import MarkdownIt
from markdown_it.token import Token
from basic_memory.markdown.plugins import observation_plugin, relation_plugin
from basic_memory.markdown.plugins import (
observation_plugin,
relation_plugin,
is_observation,
is_explicit_relation,
parse_relation,
parse_inline_relations,
)
def test_observation_plugin():
"""Test observation plugin parsing."""
"""Test observation plugin."""
# Set up markdown-it instance
md = MarkdownIt().use(observation_plugin)
# Basic observation
tokens = md.parse("- [design] Core feature #important #mvp")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert obs['category'] == 'design'
assert obs['content'] == 'Core feature #important #mvp'
assert set(obs['tags']) == {'important', 'mvp'}
assert obs['context'] is None
# With context
tokens = md.parse("- [feature] Authentication system #security (Required for MVP)")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert obs['category'] == 'feature'
assert obs['content'] == 'Authentication system #security'
assert set(obs['tags']) == {'security'}
assert obs['context'] == 'Required for MVP'
# Without category
tokens = md.parse("- Authentication system #security (Required for MVP)")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert obs['category'] is None
assert obs['content'] == 'Authentication system #security'
assert set(obs['tags']) == {'security'}
assert obs['context'] == 'Required for MVP'
# Test basic observation with all features
content = dedent("""
- [design] Basic observation #tag1 #tag2 (with context)
""")
tokens = md.parse(content)
token = [t for t in tokens if t.type == "inline"][0]
assert "observation" in token.meta
obs = token.meta["observation"]
assert obs["category"] == "design"
assert obs["content"] == "Basic observation #tag1 #tag2"
assert set(obs["tags"]) == {"tag1", "tag2"}
assert obs["context"] == "with context"
# Test without category
content = "- Basic observation #tag1 (context)"
token = [t for t in md.parse(content) if t.type == "inline"][0]
obs = token.meta["observation"]
assert obs["category"] is None
assert obs["content"] == "Basic observation #tag1"
assert obs["tags"] == ["tag1"]
assert obs["context"] == "context"
# Test without tags
content = "- [note] Basic observation (context)"
token = [t for t in md.parse(content) if t.type == "inline"][0]
obs = token.meta["observation"]
assert obs["category"] == "note"
assert obs["content"] == "Basic observation"
assert obs["tags"] is None
assert obs["context"] == "context"
def test_observation_edge_cases():
"""Test observation plugin edge cases."""
md = MarkdownIt().use(observation_plugin)
# Multiple word tags
tokens = md.parse("- [tech] Database #high-priority #needs-review")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert set(obs['tags']) == {'high-priority', 'needs-review'}
# Multiple word category
tokens = md.parse("- [user experience] Design #ux")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert obs['category'] == 'user experience'
# Parentheses in content
tokens = md.parse("- [code] Function (x) returns y #function")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert obs['content'] == 'Function (x) returns y #function'
assert obs['context'] is None
# Multiple hashtags together
tokens = md.parse("- [test] Feature #important#urgent#now")
obs_token = next(t for t in tokens if t.meta and 'observation' in t.meta)
obs = obs_token.meta['observation']
assert set(obs['tags']) == {'important', 'urgent', 'now'}
"""Test observation parsing edge cases."""
# Test non-inline token
token = Token("paragraph", "", 0)
assert not is_observation(token)
# Test empty content
token = Token("inline", "", 0)
assert not is_observation(token)
# Test markdown task
token = Token("inline", "[ ] Task item", 0)
assert not is_observation(token)
# Test completed task
token = Token("inline", "[x] Done task", 0)
assert not is_observation(token)
# Test in-progress task
token = Token("inline", "[-] Ongoing task", 0)
assert not is_observation(token)
def test_relation_plugin():
"""Test relation plugin parsing."""
"""Test relation plugin."""
md = MarkdownIt().use(relation_plugin)
# Basic relation
tokens = md.parse("- implements [[Auth Service]]")
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
rel = rel_token.meta['relations'][0]
assert rel['type'] == 'implements'
assert rel['target'] == 'Auth Service'
assert rel['context'] is None
# With context
tokens = md.parse("- depends_on [[Database]] (Required for persistence)")
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
rel = rel_token.meta['relations'][0]
assert rel['type'] == 'depends_on'
assert rel['target'] == 'Database'
assert rel['context'] == 'Required for persistence'
# Test explicit relation with all features
content = dedent("""
- implements [[Component]] (with context)
""")
tokens = md.parse(content)
token = [t for t in tokens if t.type == "inline"][0]
assert "relations" in token.meta
rel = token.meta["relations"][0]
assert rel["type"] == "implements"
assert rel["target"] == "Component"
assert rel["context"] == "with context"
# Test implicit relations in text
content = "Some text with a [[Link]] and [[Another Link]]"
token = [t for t in md.parse(content) if t.type == "inline"][0]
rels = token.meta["relations"]
assert len(rels) == 2
assert rels[0]["type"] == "links to"
assert rels[0]["target"] == "Link"
assert rels[1]["target"] == "Another Link"
def test_relation_edge_cases():
"""Test relation plugin edge cases."""
md = MarkdownIt().use(relation_plugin)
# Multiple word type
tokens = md.parse("- is used by [[Client App]] (Primary consumer)")
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
rel = rel_token.meta['relations'][0]
assert rel['type'] == 'is used by'
# Extra spaces
tokens = md.parse("- specifies [[Format]] (Documentation)")
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
rel = rel_token.meta['relations'][0]
assert rel['type'] == 'specifies'
assert rel['target'] == 'Format'
"""Test relation parsing edge cases."""
# Test non-inline token
token = Token("paragraph", "", 0)
assert not is_explicit_relation(token)
# Test empty content
token = Token("inline", "", 0)
assert not is_explicit_relation(token)
def test_inline_relations():
"""Test finding relations in regular content."""
md = MarkdownIt().use(relation_plugin)
# Single inline link
tokens = md.parse("This references [[Another Doc]].")
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
rel = rel_token.meta['relations'][0]
assert rel['type'] == 'links to'
assert rel['target'] == 'Another Doc'
# Multiple inline links
tokens = md.parse("Links to [[Doc1]] and [[Doc2]].")
rel_token = next(t for t in tokens if t.meta and 'relations' in t.meta)
rels = rel_token.meta['relations']
assert len(rels) == 2
assert {r['target'] for r in rels} == {'Doc1', 'Doc2'}
# Test incomplete relation (missing target)
token = Token("inline", "relates_to [[]]", 0)
result = parse_relation(token)
assert result is None
# Test non-relation content
token = Token("inline", "Just some text", 0)
result = parse_relation(token)
assert result is None
# Test invalid inline link (empty target)
assert not parse_inline_relations("Text with [[]] empty link")
# Test nested links (avoid duplicates)
result = parse_inline_relations("Text with [[Outer [[Inner]] Link]]")
assert len(result) == 1
assert result[0]["target"] == "Outer [[Inner]] Link"
def test_combined_plugins():
"""Test both plugins working together."""
md = (MarkdownIt()
.use(observation_plugin)
.use(relation_plugin))
content = """# Document
md = MarkdownIt().use(observation_plugin).use(relation_plugin)
Some text with a [[Link]].
content = dedent("""
# Section
- [design] Observation with [[Link]] #tag (context)
- implements [[Component]] (details)
- Just a [[Reference]] in text
Some text with a [[Link]] reference.
""")
## Observations
- [tech] Implements [[Feature]] #important
- [design] References [[AnotherDoc]] (For context)
"""
tokens = md.parse(content)
# Should find both observations and relations
observations = [t.meta['observation'] for t in tokens if t.meta and 'observation' in t.meta]
relations = [r for t in tokens if t.meta and 'relations' in t.meta for r in t.meta['relations']]
assert len(observations) == 2
assert any(o['category'] == 'tech' and 'important' in o['tags'] for o in observations)
assert any(o['category'] == 'design' and o['context'] == 'For context' for o in observations)
# Should find all relations (Feature, AnotherDoc, and Link)
assert len(relations) == 3
assert any(r['target'] == 'Link' and r['type'] == 'links to' for r in relations)
assert any(r['target'] == 'Feature' for r in relations)
assert any(r['target'] == 'AnotherDoc' for r in relations)
inline_tokens = [t for t in tokens if t.type == "inline"]
# First token has both observation and relation
obs_token = inline_tokens[1]
assert "observation" in obs_token.meta
assert "relations" in obs_token.meta
# Second token has explicit relation
rel_token = inline_tokens[2]
assert "relations" in rel_token.meta
rel = rel_token.meta["relations"][0]
assert rel["type"] == "implements"
# Third token has implicit relation
text_token = inline_tokens[4]
assert "relations" in text_token.meta
link = text_token.meta["relations"][0]
assert link["type"] == "links to"
+3 -3
View File
@@ -38,7 +38,7 @@ async def test_write_new_minimal_file(markdown_processor: MarkdownProcessor, tmp
)
# Write file
checksum = await markdown_processor.write_file(path, markdown)
await markdown_processor.write_file(path, markdown)
# Read back and verify
content = path.read_text()
@@ -87,7 +87,7 @@ async def test_write_new_file_with_content(markdown_processor: MarkdownProcessor
)
# Write file
checksum = await markdown_processor.write_file(path, markdown)
await markdown_processor.write_file(path, markdown)
# Read back and verify
content = path.read_text()
@@ -135,7 +135,7 @@ async def test_update_preserves_content(markdown_processor: MarkdownProcessor, t
)
# Update file
new_checksum = await markdown_processor.write_file(path, updated, expected_checksum=checksum)
await markdown_processor.write_file(path, updated, expected_checksum=checksum)
# Read back and verify
result = await markdown_processor.read_file(path)
@@ -91,7 +91,6 @@ def test_no_category():
assert observation.content == "No category"
def test_unicode_content():
"""Test handling of Unicode content."""
md = MarkdownIt().use(observation_plugin)

Some files were not shown because too many files have changed in this diff Show More