mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
100 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 132c5671d8 | |||
| 71de8acfd0 | |||
| 50469d691b | |||
| 85e620c72b | |||
| 493e0902b6 | |||
| 56f47d6812 | |||
| a15c346d5e | |||
| c06cf19e62 | |||
| 41f5b1667b | |||
| ca632beb6f | |||
| a491c2b7d4 | |||
| 674dd1fd47 | |||
| a91da13967 | |||
| e0803734e6 | |||
| 8a5a970ba9 | |||
| 65ebe5d194 | |||
| 8370a06b6c | |||
| 3270e6d82b | |||
| c05d363a99 | |||
| 3f9c68ed85 | |||
| a8ffe051ea | |||
| 2c74434e99 | |||
| bea3d6c889 | |||
| 14539010b1 | |||
| a19287c967 | |||
| 64430b850e | |||
| 10b2e91e34 | |||
| 1989db15fa | |||
| 17f0517917 | |||
| 8de84c0221 | |||
| b1d2a64933 | |||
| fe8c3d87b0 | |||
| 89ee324df0 | |||
| 8664c57bb3 | |||
| 8117a7b0ed | |||
| f47fb2aeef | |||
| 8fa197e2ec | |||
| 58f3fe95b7 | |||
| 1d6054d30a | |||
| 812947fc4b | |||
| dab957314a | |||
| 70b2eb3a4a | |||
| 80cdecdbf0 | |||
| 8dd923d5bc | |||
| 0cb429bfd3 | |||
| a15265783e | |||
| e48deaeecf | |||
| 854cf8302e | |||
| 6e4a53cdfa | |||
| 2e215fe83c | |||
| b546941779 | |||
| 29a259421a | |||
| 071e76a0ff | |||
| 9ee67e9de0 | |||
| edbc04be60 | |||
| 864209c5f8 | |||
| b40f0c75da | |||
| 230738ee9c | |||
| d15d8f91a4 | |||
| a6fdfad374 | |||
| 3e78fcc2c2 | |||
| b5d09a4854 | |||
| 01d46727b4 | |||
| 713b3bedf8 | |||
| 97d3a0196e | |||
| 052f491fff | |||
| 6589f5d251 | |||
| 9a4070eccf | |||
| 41d4d81c1a | |||
| 2bc4847c98 | |||
| 4b1be9e54f | |||
| 53d220df58 | |||
| 128b7657ec | |||
| 373640c843 | |||
| d4c8293687 | |||
| 0d48404af4 | |||
| f11bf78f3f | |||
| 31c55c7a32 | |||
| 7c381a59c9 | |||
| 40a8242002 | |||
| dde9ff228b | |||
| 2f9178b050 | |||
| eb3360cc22 | |||
| f1eeaee104 | |||
| 355519a786 | |||
| 23f45ce788 | |||
| 86380a92b2 | |||
| a87ac58646 | |||
| 28513c8b8d | |||
| c394d682e1 | |||
| 5b4f0eafcc | |||
| 452308fd96 | |||
| 8094528c40 | |||
| fa77a41e6f | |||
| 12cd1e43b0 | |||
| c58a2e0893 | |||
| 28318d8435 | |||
| 7f7ec67cbb | |||
| 04e575041e | |||
| 1fee436bf9 |
@@ -0,0 +1,41 @@
|
||||
name: "Pull Request Title"
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- edited
|
||||
- synchronize
|
||||
|
||||
jobs:
|
||||
main:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: amannn/action-semantic-pull-request@v5
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
# Configure allowed types based on what we want in our changelog
|
||||
types: |
|
||||
feat
|
||||
fix
|
||||
chore
|
||||
docs
|
||||
style
|
||||
refactor
|
||||
perf
|
||||
test
|
||||
build
|
||||
ci
|
||||
# Require at least one from scope list (optional)
|
||||
scopes: |
|
||||
core
|
||||
cli
|
||||
api
|
||||
mcp
|
||||
sync
|
||||
ui
|
||||
deps
|
||||
installer
|
||||
# Allow breaking changes (needs "!" after type/scope)
|
||||
requireScopeForBreakingChange: true
|
||||
@@ -20,6 +20,9 @@ jobs:
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: write
|
||||
outputs:
|
||||
released: ${{ steps.release.outputs.released }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -43,4 +46,51 @@ jobs:
|
||||
if: steps.release.outputs.released == 'true'
|
||||
with:
|
||||
github_token: ${{ secrets.GITHUB_TOKEN }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
tag: ${{ steps.release.outputs.tag }}
|
||||
|
||||
build-macos:
|
||||
needs: release
|
||||
if: needs.release.outputs.released == 'true'
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ needs.release.outputs.tag }}
|
||||
|
||||
- name: Set up Python "3.12"
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install librsvg
|
||||
run: brew install librsvg
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
pip install uv
|
||||
|
||||
- name: Create virtual env
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
uv sync
|
||||
|
||||
- name: Build macOS installer
|
||||
run: |
|
||||
make installer-mac
|
||||
xattr -dr com.apple.quarantine "installer/build/Basic Memory Installer.app"
|
||||
|
||||
- name: Zip macOS installer
|
||||
run: |
|
||||
cd installer/build
|
||||
zip -ry "Basic-Memory-Installer-${{ needs.release.outputs.tag }}.zip" "Basic Memory Installer.app"
|
||||
|
||||
- name: Upload macOS installer
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: installer/build/Basic-Memory-Installer-${{ needs.release.outputs.tag }}.zip
|
||||
tag_name: ${{ needs.release.outputs.tag }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
@@ -37,6 +37,10 @@ jobs:
|
||||
run: |
|
||||
uv pip install -e .[dev]
|
||||
|
||||
- name: Run type checks
|
||||
run: |
|
||||
uv run make type-check
|
||||
|
||||
- name: Run tests
|
||||
run: |
|
||||
uv pip install pytest pytest-cov
|
||||
|
||||
+16
-13
@@ -1,8 +1,10 @@
|
||||
# Python
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
*$py.class
|
||||
*.so
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
|
||||
# Distribution / packaging
|
||||
.Python
|
||||
build/
|
||||
develop-eggs/
|
||||
@@ -20,7 +22,12 @@ wheels/
|
||||
.installed.cfg
|
||||
*.egg
|
||||
|
||||
# Virtual Environment
|
||||
# Installer artifacts
|
||||
installer/build/
|
||||
installer/dist/
|
||||
rw.*.dmg # Temporary disk images
|
||||
|
||||
# Virtual environments
|
||||
.env
|
||||
.venv
|
||||
env/
|
||||
@@ -30,13 +37,9 @@ ENV/
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
|
||||
# Project specific
|
||||
projects/*.db
|
||||
projects/*.db-journal
|
||||
/.coverage
|
||||
|
||||
**/.DS_Store
|
||||
|
||||
*.log
|
||||
# macOS
|
||||
.DS_Store
|
||||
/.coverage.*
|
||||
|
||||
+246
@@ -1,6 +1,249 @@
|
||||
# CHANGELOG
|
||||
|
||||
|
||||
## v0.4.1 (2025-02-17)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix alemic config
|
||||
([`71de8ac`](https://github.com/basicmachines-co/basic-memory/commit/71de8acfd0902fc60f27deb3638236a3875787ab))
|
||||
|
||||
|
||||
## v0.4.0 (2025-02-16)
|
||||
|
||||
### Features
|
||||
|
||||
- Import chatgpt conversation data ([#9](https://github.com/basicmachines-co/basic-memory/pull/9),
|
||||
[`56f47d6`](https://github.com/basicmachines-co/basic-memory/commit/56f47d6812982437f207629e6ac9a82e0e56514e))
|
||||
|
||||
Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
- Import claude.ai data ([#8](https://github.com/basicmachines-co/basic-memory/pull/8),
|
||||
[`a15c346`](https://github.com/basicmachines-co/basic-memory/commit/a15c346d5ebd44344b76bad877bb4d1073fcbc3b))
|
||||
|
||||
Import Claude.ai conversation and project data to basic-memory Markdown format.
|
||||
|
||||
---------
|
||||
|
||||
Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
|
||||
## v0.3.0 (2025-02-15)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refactor db schema migrate handling
|
||||
([`ca632be`](https://github.com/basicmachines-co/basic-memory/commit/ca632beb6fed5881f4d8ba5ce698bb5bc681e6aa))
|
||||
|
||||
|
||||
## v0.2.21 (2025-02-15)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix osx installer github action
|
||||
([`65ebe5d`](https://github.com/basicmachines-co/basic-memory/commit/65ebe5d19491e5ff047c459d799498ad5dd9cd1a))
|
||||
|
||||
- Handle memory:// url format in read_note tool
|
||||
([`e080373`](https://github.com/basicmachines-co/basic-memory/commit/e0803734e69eeb6c6d7432eea323c7a264cb8347))
|
||||
|
||||
- Remove create schema from init_db
|
||||
([`674dd1f`](https://github.com/basicmachines-co/basic-memory/commit/674dd1fd47be9e60ac17508476c62254991df288))
|
||||
|
||||
### Features
|
||||
|
||||
- Set version in var, output version at startup
|
||||
([`a91da13`](https://github.com/basicmachines-co/basic-memory/commit/a91da1396710e62587df1284da00137d156fc05e))
|
||||
|
||||
|
||||
## v0.2.20 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix installer artifact
|
||||
([`8de84c0`](https://github.com/basicmachines-co/basic-memory/commit/8de84c0221a1ee32780aa84dac4d3ea60895e05c))
|
||||
|
||||
|
||||
## v0.2.19 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Get app artifact for installer
|
||||
([`fe8c3d8`](https://github.com/basicmachines-co/basic-memory/commit/fe8c3d87b003166252290a87cbe958301cccf797))
|
||||
|
||||
|
||||
## v0.2.18 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Don't zip app on release
|
||||
([`8664c57`](https://github.com/basicmachines-co/basic-memory/commit/8664c57bb331d7f3f7e0239acb5386c7a3c6144e))
|
||||
|
||||
|
||||
## v0.2.17 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix app zip in installer release
|
||||
([`8fa197e`](https://github.com/basicmachines-co/basic-memory/commit/8fa197e2ec8a1b6caaf6dbb39c3c6626bba23e2e))
|
||||
|
||||
|
||||
## v0.2.16 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Debug inspect build on ci
|
||||
([`1d6054d`](https://github.com/basicmachines-co/basic-memory/commit/1d6054d30a477a4e6a5d6ac885632e50c01945d3))
|
||||
|
||||
|
||||
## v0.2.15 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Debug installer ci
|
||||
([`dab9573`](https://github.com/basicmachines-co/basic-memory/commit/dab957314aec9ed0e12abca2265552494ae733a2))
|
||||
|
||||
|
||||
## v0.2.14 (2025-02-14)
|
||||
|
||||
|
||||
## v0.2.13 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refactor release.yml installer
|
||||
([`a152657`](https://github.com/basicmachines-co/basic-memory/commit/a15265783e47c22d8c7931396281d023b3694e27))
|
||||
|
||||
- Try using symlinks in installer build
|
||||
([`8dd923d`](https://github.com/basicmachines-co/basic-memory/commit/8dd923d5bc0587276f92b5f1db022ad9c8687e45))
|
||||
|
||||
|
||||
## v0.2.12 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix cx_freeze options for installer
|
||||
([`854cf83`](https://github.com/basicmachines-co/basic-memory/commit/854cf8302e2f83578030db05e29b8bdc4348795a))
|
||||
|
||||
|
||||
## v0.2.11 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Ci installer app fix #37
|
||||
([`2e215fe`](https://github.com/basicmachines-co/basic-memory/commit/2e215fe83ca421b921186c7f1989dc2cb5cca278))
|
||||
|
||||
|
||||
## v0.2.10 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix build on github ci for app installer
|
||||
([`29a2594`](https://github.com/basicmachines-co/basic-memory/commit/29a259421a0ccb10cfa68e3707eaa506ad5e55c0))
|
||||
|
||||
|
||||
## v0.2.9 (2025-02-14)
|
||||
|
||||
|
||||
## v0.2.8 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix installer on ci, maybe
|
||||
([`edbc04b`](https://github.com/basicmachines-co/basic-memory/commit/edbc04be601d234bb1f5eb3ba24d6ad55244b031))
|
||||
|
||||
|
||||
## v0.2.7 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Try to fix installer ci
|
||||
([`230738e`](https://github.com/basicmachines-co/basic-memory/commit/230738ee9c110c0509e0a09cb0e101a92cfcb729))
|
||||
|
||||
|
||||
## v0.2.6 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Bump project patch version
|
||||
([`01d4672`](https://github.com/basicmachines-co/basic-memory/commit/01d46727b40c24b017ea9db4b741daef565ac73e))
|
||||
|
||||
- Fix installer setup.py change ci to use make
|
||||
([`3e78fcc`](https://github.com/basicmachines-co/basic-memory/commit/3e78fcc2c208d83467fe7199be17174d7ffcad1a))
|
||||
|
||||
|
||||
## v0.2.5 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refix vitual env in installer build
|
||||
([`052f491`](https://github.com/basicmachines-co/basic-memory/commit/052f491fff629e8ead629c9259f8cb46c608d584))
|
||||
|
||||
|
||||
## v0.2.4 (2025-02-14)
|
||||
|
||||
|
||||
## v0.2.3 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Workaround unsigned app
|
||||
([`41d4d81`](https://github.com/basicmachines-co/basic-memory/commit/41d4d81c1ad1dc2923ba0e903a57454a0c8b6b5c))
|
||||
|
||||
|
||||
## v0.2.2 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix path to intaller app artifact
|
||||
([`53d220d`](https://github.com/basicmachines-co/basic-memory/commit/53d220df585561f9edd0d49a9e88f1d4055059cf))
|
||||
|
||||
|
||||
## v0.2.1 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Activate vitualenv in installer build
|
||||
([`d4c8293`](https://github.com/basicmachines-co/basic-memory/commit/d4c8293687a52eaf3337fe02e2f7b80e4cc9a1bb))
|
||||
|
||||
- Trigger installer build on release
|
||||
([`f11bf78`](https://github.com/basicmachines-co/basic-memory/commit/f11bf78f3f600d0e1b01996cf8e1f9c39e3dd218))
|
||||
|
||||
|
||||
## v0.2.0 (2025-02-14)
|
||||
|
||||
### Features
|
||||
|
||||
- Build installer via github action ([#7](https://github.com/basicmachines-co/basic-memory/pull/7),
|
||||
[`7c381a5`](https://github.com/basicmachines-co/basic-memory/commit/7c381a59c962053c78da096172e484f28ab47e96))
|
||||
|
||||
* feat(ci): build installer via github action
|
||||
|
||||
* enforce conventional commits in PR titles
|
||||
|
||||
* feat: add icon to installer
|
||||
|
||||
---------
|
||||
|
||||
Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
|
||||
## v0.1.2 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix installer for mac
|
||||
([`dde9ff2`](https://github.com/basicmachines-co/basic-memory/commit/dde9ff228b72852b5abc58faa1b5e7c6f8d2c477))
|
||||
|
||||
- Remove unused FileChange dataclass
|
||||
([`eb3360c`](https://github.com/basicmachines-co/basic-memory/commit/eb3360cc221f892b12a17137ae740819d48248e8))
|
||||
|
||||
- Update uv installer url
|
||||
([`2f9178b`](https://github.com/basicmachines-co/basic-memory/commit/2f9178b0507b3b69207d5c80799f2d2f573c9a04))
|
||||
|
||||
|
||||
## v0.1.1 (2025-02-07)
|
||||
|
||||
|
||||
## v0.1.0 (2025-02-07)
|
||||
|
||||
### Bug Fixes
|
||||
@@ -17,6 +260,9 @@
|
||||
- Install fastapi deps after removing basic-foundation
|
||||
([`51a741e`](https://github.com/basicmachines-co/basic-memory/commit/51a741e7593a1ea0e5eb24e14c70ff61670f9663))
|
||||
|
||||
- Recreate search index on db reset
|
||||
([`1fee436`](https://github.com/basicmachines-co/basic-memory/commit/1fee436bf903a35c9ebb7d87607fc9cc9f5ff6e7))
|
||||
|
||||
- Remove basic-foundation from deps
|
||||
([`b8d0c71`](https://github.com/basicmachines-co/basic-memory/commit/b8d0c7160f29c97cdafe398a7e6a5240473e0c89))
|
||||
|
||||
|
||||
@@ -1,39 +1,43 @@
|
||||
.PHONY: install test lint db-new db-up db-down db-reset
|
||||
.PHONY: install test lint clean format type-check installer-mac installer-win
|
||||
|
||||
install:
|
||||
brew install dbmate
|
||||
pip install -e ".[dev]"
|
||||
|
||||
test:
|
||||
pytest -p pytest_mock -v
|
||||
|
||||
lint:
|
||||
black .
|
||||
ruff check .
|
||||
ruff check . --fix
|
||||
|
||||
db-new:
|
||||
dbmate new $(name)
|
||||
|
||||
db-up:
|
||||
dbmate up
|
||||
|
||||
db-down:
|
||||
dbmate down
|
||||
|
||||
db-reset:
|
||||
dbmate drop
|
||||
dbmate up
|
||||
type-check:
|
||||
uv run pyright
|
||||
|
||||
clean:
|
||||
find . -type f -name '*.pyc' -delete
|
||||
find . -type d -name '__pycache__' -exec rm -r {} +
|
||||
rm -rf installer/build/
|
||||
rm -rf installer/dist/
|
||||
rm -f rw.*.dmg
|
||||
rm -rf dist
|
||||
rm -rf installer/build
|
||||
rm -rf installer/dist
|
||||
rm -f .coverage.*
|
||||
|
||||
format:
|
||||
uv run ruff format .
|
||||
|
||||
format: format-python
|
||||
#format: format-python format-prettier
|
||||
|
||||
# run inspector tool
|
||||
run-dev:
|
||||
uv run mcp dev src/basic_memory/mcp/main.py
|
||||
uv run mcp dev src/basic_memory/mcp/main.py
|
||||
|
||||
# Build app installer
|
||||
installer-mac:
|
||||
cd installer && chmod +x make_icons.sh && ./make_icons.sh
|
||||
cd installer && uv run python setup.py bdist_mac
|
||||
|
||||
installer-win:
|
||||
cd installer && uv run python setup.py bdist_win32
|
||||
|
||||
|
||||
update-deps:
|
||||
uv lock f--upgrade
|
||||
|
||||
@@ -253,6 +253,92 @@ Basic Memory is built on some key ideas:
|
||||
- Simple text patterns can capture rich meaning
|
||||
- Local-first doesn't mean feature-poor
|
||||
|
||||
## Importing data
|
||||
|
||||
Basic memory has cli commands to import data from several formats into Markdown files
|
||||
|
||||
### Claude.ai
|
||||
|
||||
First, request an export of your data from your Claude account. The data will be emailed to you in several files,
|
||||
including
|
||||
`conversations.json` and `projects.json`.
|
||||
|
||||
Import Claude.ai conversation data
|
||||
|
||||
```bash
|
||||
basic-memory import claude conversations
|
||||
```
|
||||
|
||||
The conversations will be turned into Markdown files and placed in the "conversations" folder by default (this can be
|
||||
changed with the --folder arg).
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
Importing chats from conversations.json...writing to .../basic-memory
|
||||
Reading chat data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭────────────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Imported 307 conversations │
|
||||
│ Containing 7769 messages │
|
||||
╰────────────────────────────╯
|
||||
```
|
||||
|
||||
Next, you can run the `sync` command to import the data into basic-memory
|
||||
|
||||
```bash
|
||||
basic-memory sync
|
||||
```
|
||||
|
||||
You can also import project data from Claude.ai
|
||||
|
||||
```bash
|
||||
➜ basic-memory import claude projects
|
||||
Importing projects from projects.json...writing to .../basic-memory/projects
|
||||
Reading project data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭────────────────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Imported 101 project documents │
|
||||
│ Imported 32 prompt templates │
|
||||
╰────────────────────────────────╯
|
||||
|
||||
Run 'basic-memory sync' to index the new files.
|
||||
```
|
||||
|
||||
### Chat Gpt
|
||||
|
||||
```bash
|
||||
➜ basic-memory import chatgpt
|
||||
Importing chats from conversations.json...writing to .../basic-memory/conversations
|
||||
|
||||
Reading chat data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭────────────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Imported 198 conversations │
|
||||
│ Containing 11777 messages │
|
||||
╰────────────────────────────╯
|
||||
|
||||
|
||||
```
|
||||
|
||||
### Memory json
|
||||
|
||||
```bash
|
||||
➜ basic-memory import memory-json
|
||||
Importing from memory.json...writing to .../basic-memory
|
||||
Reading memory.json... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
Creating entities... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭──────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Created 126 entities │
|
||||
│ Added 252 relations │
|
||||
╰──────────────────────╯
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
AGPL-3.0
|
||||
Binary file not shown.
@@ -0,0 +1,26 @@
|
||||
# Basic Memory Installer
|
||||
|
||||
This installer configures Basic Memory to work with Claude Desktop.
|
||||
|
||||
## Installation
|
||||
|
||||
1. Download the latest installer from the [releases page](https://github.com/basicmachines-co/basic-memory/releases)
|
||||
2. Unzip the downloaded file
|
||||
3. Since the app is currently unsigned, you'll need to:
|
||||
|
||||
On your Mac, choose Apple menu > System Settings, then click Privacy & Security in the sidebar. (You may need to
|
||||
scroll down.)
|
||||
|
||||
Go to Security, then click Open.
|
||||
|
||||
Click Open Anyway.
|
||||
|
||||
This button is available for about an hour after you try to open the app.
|
||||
|
||||
Enter your login password, then click OK.
|
||||
|
||||
https://support.apple.com/guide/mac-help/apple-cant-check-app-for-malicious-software-mchleab3a043/mac
|
||||
|
||||
5. Restart Claude Desktop
|
||||
|
||||
The warning only appears the first time you open the app. Future updates will include proper code signing.
|
||||
@@ -0,0 +1,64 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<svg viewBox="0 0 512 512" xmlns="http://www.w3.org/2000/svg">
|
||||
<!-- Background -->
|
||||
<rect x="0" y="0" width="512" height="512" rx="64" fill="#111111"/>
|
||||
|
||||
<!-- Define arrowhead marker -->
|
||||
<defs>
|
||||
<marker id="arrowhead"
|
||||
markerWidth="10"
|
||||
markerHeight="10"
|
||||
refX="8"
|
||||
refY="5"
|
||||
orient="auto">
|
||||
<path d="M 0 0 L 10 5 L 0 10 Z"
|
||||
fill="#00cc00"/>
|
||||
</marker>
|
||||
</defs>
|
||||
|
||||
<!-- State 1 (initial) -->
|
||||
<circle cx="156" cy="256" r="30" fill="none" stroke="#00cc00" stroke-width="3"/>
|
||||
|
||||
<!-- State 2 (accept) -->
|
||||
<circle cx="356" cy="176" r="34" fill="none" stroke="#00cc00" stroke-width="3"/>
|
||||
<circle cx="356" cy="176" r="28" fill="none" stroke="#00cc00" stroke-width="3"/>
|
||||
|
||||
<!-- State 3 (accept) -->
|
||||
<circle cx="356" cy="336" r="34" fill="none" stroke="#00cc00" stroke-width="3"/>
|
||||
<circle cx="356" cy="336" r="28" fill="none" stroke="#00cc00" stroke-width="3"/>
|
||||
|
||||
<!-- Initial arrow -->
|
||||
<path d="M 96 256 L 126 256"
|
||||
stroke="#00cc00" stroke-width="3" fill="none"
|
||||
marker-end="url(#arrowhead)"/>
|
||||
|
||||
<!-- State transitions -->
|
||||
<!-- 1 -> 2 -->
|
||||
<path d="M 180 240
|
||||
Q 260 200, 320 176"
|
||||
stroke="#00cc00" stroke-width="3" fill="none"
|
||||
marker-end="url(#arrowhead)"/>
|
||||
|
||||
<!-- 1 -> 3 -->
|
||||
<path d="M 180 272
|
||||
Q 260 312, 320 336"
|
||||
stroke="#00cc00" stroke-width="3" fill="none"
|
||||
marker-end="url(#arrowhead)"/>
|
||||
|
||||
<!-- Self loops -->
|
||||
<path d="M 356 142
|
||||
Q 396 142, 396 176
|
||||
Q 396 210, 356 210
|
||||
Q 316 210, 316 176
|
||||
Q 316 142, 356 142"
|
||||
stroke="#00cc00" stroke-width="2" fill="none"
|
||||
marker-end="url(#arrowhead)"/>
|
||||
|
||||
<path d="M 356 302
|
||||
Q 396 302, 396 336
|
||||
Q 396 370, 356 370
|
||||
Q 316 370, 316 336
|
||||
Q 316 302, 356 302"
|
||||
stroke="#00cc00" stroke-width="2" fill="none"
|
||||
marker-end="url(#arrowhead)"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.0 KiB |
@@ -0,0 +1,90 @@
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
# Use tkinter for GUI alerts on macOS
|
||||
if sys.platform == "darwin":
|
||||
import tkinter as tk
|
||||
from tkinter import messagebox
|
||||
|
||||
|
||||
def ensure_uv_installed():
|
||||
"""Check if uv is installed, install if not."""
|
||||
try:
|
||||
subprocess.run(["uv", "--version"], capture_output=True, check=True)
|
||||
except (subprocess.CalledProcessError, FileNotFoundError):
|
||||
print("Installing uv package manager...")
|
||||
subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"-LsSf",
|
||||
"https://astral.sh/uv/install.sh",
|
||||
"|",
|
||||
"sh",
|
||||
],
|
||||
shell=True,
|
||||
)
|
||||
|
||||
|
||||
def get_config_path():
|
||||
"""Get Claude Desktop config path for current platform."""
|
||||
if sys.platform == "darwin":
|
||||
return Path.home() / "Library/Application Support/Claude/claude_desktop_config.json"
|
||||
elif sys.platform == "win32":
|
||||
return Path.home() / "AppData/Roaming/Claude/claude_desktop_config.json"
|
||||
else:
|
||||
raise RuntimeError(f"Unsupported platform: {sys.platform}")
|
||||
|
||||
|
||||
def update_claude_config():
|
||||
"""Update Claude Desktop config to include basic-memory."""
|
||||
config_path = get_config_path()
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load existing config or create new
|
||||
if config_path.exists():
|
||||
config = json.loads(config_path.read_text())
|
||||
else:
|
||||
config = {"mcpServers": {}}
|
||||
|
||||
# Add/update basic-memory config
|
||||
config["mcpServers"]["basic-memory"] = {"command": "uvx", "args": ["basic-memory", "mcp"]}
|
||||
|
||||
# Write back config
|
||||
config_path.write_text(json.dumps(config, indent=2))
|
||||
|
||||
|
||||
def print_completion_message():
|
||||
"""Show completion message with helpful tips."""
|
||||
message = """Installation complete! Basic Memory is now available in Claude Desktop.
|
||||
|
||||
Please restart Claude Desktop for changes to take effect.
|
||||
|
||||
Quick Start:
|
||||
1. You can run sync directly using: uvx basic-memory sync
|
||||
2. Optionally, install globally with: uv pip install basic-memory
|
||||
|
||||
Built with ♥️ by Basic Machines."""
|
||||
|
||||
if sys.platform == "darwin":
|
||||
# Show GUI message on macOS
|
||||
root = tk.Tk()
|
||||
root.withdraw() # Hide the main window
|
||||
messagebox.showinfo("Basic Memory", message)
|
||||
root.destroy()
|
||||
else:
|
||||
# Fallback to console output
|
||||
print(message)
|
||||
|
||||
|
||||
def main():
|
||||
print("Welcome to Basic Memory installer")
|
||||
ensure_uv_installed()
|
||||
print("Configuring Claude Desktop...")
|
||||
update_claude_config()
|
||||
print_completion_message()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Convert SVG to PNG at various required sizes
|
||||
rsvg-convert -h 16 -w 16 icon.svg > icon_16x16.png
|
||||
rsvg-convert -h 32 -w 32 icon.svg > icon_32x32.png
|
||||
rsvg-convert -h 128 -w 128 icon.svg > icon_128x128.png
|
||||
rsvg-convert -h 256 -w 256 icon.svg > icon_256x256.png
|
||||
rsvg-convert -h 512 -w 512 icon.svg > icon_512x512.png
|
||||
|
||||
# Create iconset directory
|
||||
mkdir -p Basic.iconset
|
||||
|
||||
# Move files into iconset with Mac-specific names
|
||||
cp icon_16x16.png Basic.iconset/icon_16x16.png
|
||||
cp icon_32x32.png Basic.iconset/icon_16x16@2x.png
|
||||
cp icon_32x32.png Basic.iconset/icon_32x32.png
|
||||
cp icon_128x128.png Basic.iconset/icon_32x32@2x.png
|
||||
cp icon_256x256.png Basic.iconset/icon_128x128.png
|
||||
cp icon_512x512.png Basic.iconset/icon_256x256.png
|
||||
cp icon_512x512.png Basic.iconset/icon_512x512.png
|
||||
|
||||
# Convert iconset to icns
|
||||
iconutil -c icns Basic.iconset
|
||||
|
||||
# Clean up
|
||||
rm -rf Basic.iconset
|
||||
rm icon_*.png
|
||||
@@ -0,0 +1,40 @@
|
||||
from cx_Freeze import setup, Executable
|
||||
import sys
|
||||
|
||||
# Build options for all platforms
|
||||
build_exe_options = {
|
||||
"packages": ["json", "pathlib"],
|
||||
"excludes": ["unittest", "pydoc", "test"],
|
||||
}
|
||||
|
||||
# Platform-specific options
|
||||
if sys.platform == "win32":
|
||||
base = "Win32GUI" # Use GUI base for Windows
|
||||
build_exe_options.update(
|
||||
{
|
||||
"include_msvcr": True,
|
||||
}
|
||||
)
|
||||
target_name = "Basic Memory Installer.exe"
|
||||
else: # darwin
|
||||
base = None # Don't use GUI base for macOS
|
||||
target_name = "Basic Memory Installer"
|
||||
|
||||
executables = [
|
||||
Executable(script="installer.py", target_name=target_name, base=base, icon="Basic.icns")
|
||||
]
|
||||
|
||||
setup(
|
||||
name="basic-memory",
|
||||
version=open("../pyproject.toml").read().split('version = "', 1)[1].split('"', 1)[0],
|
||||
description="Basic Memory - Local-first knowledge management",
|
||||
options={
|
||||
"build_exe": build_exe_options,
|
||||
"bdist_mac": {
|
||||
"bundle_name": "Basic Memory Installer",
|
||||
"iconfile": "Basic.icns",
|
||||
"codesign_identity": "-", # Force ad-hoc signing
|
||||
},
|
||||
},
|
||||
executables=executables,
|
||||
)
|
||||
+20
-11
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "basic-memory"
|
||||
version = "0.1.0"
|
||||
version = "0.4.1"
|
||||
description = "Local-first knowledge management combining Zettelkasten with knowledge graphs"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12.1"
|
||||
@@ -28,16 +28,9 @@ dependencies = [
|
||||
"watchfiles>=1.0.4",
|
||||
"fastapi[standard]>=0.115.8",
|
||||
"alembic>=1.14.1",
|
||||
"qasync>=0.27.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
dev = [
|
||||
"pytest>=8.3.4",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-mock>=3.12.0",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"ruff>=0.1.6",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/basicmachines-co/basic-memory"
|
||||
@@ -64,10 +57,22 @@ target-version = "py312"
|
||||
|
||||
[tool.uv]
|
||||
dev-dependencies = [
|
||||
"gevent>=24.11.1",
|
||||
"icecream>=2.1.3",
|
||||
"pytest>=8.3.4",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-mock>=3.12.0",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"ruff>=0.1.6",
|
||||
"pytest>=8.3.4",
|
||||
"pytest-cov>=4.1.0",
|
||||
"pytest-mock>=3.12.0",
|
||||
"pytest-asyncio>=0.24.0",
|
||||
"ruff>=0.1.6",
|
||||
"cx-freeze>=7.2.10",
|
||||
"pyqt6>=6.8.1",
|
||||
]
|
||||
|
||||
|
||||
[tool.pyright]
|
||||
include = ["src/"]
|
||||
exclude = ["**/__pycache__"]
|
||||
@@ -79,7 +84,9 @@ pythonVersion = "3.12"
|
||||
|
||||
|
||||
[tool.semantic_release]
|
||||
version_variable = "src/basic_memory/__init__.py:__version__"
|
||||
version_variables = [
|
||||
"src/basic_memory/__init__.py:__version__",
|
||||
]
|
||||
version_toml = [
|
||||
"pyproject.toml:project.version",
|
||||
]
|
||||
@@ -91,3 +98,5 @@ dist_path = "dist/"
|
||||
upload_to_pypi = true
|
||||
commit_message = "chore(release): {version} [skip ci]"
|
||||
|
||||
[tool.coverage.run]
|
||||
concurrency = ["thread", "gevent"]
|
||||
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "Welcome to Basic Memory installer"
|
||||
|
||||
# 1. Install uv if not present
|
||||
if ! command -v uv &> /dev/null; then
|
||||
echo "Installing uv package manager..."
|
||||
curl -LsSf https://github.com/astral-sh/uv/releases/download/0.1.23/uv-installer.sh | sh
|
||||
fi
|
||||
|
||||
# 2. Configure Claude Desktop
|
||||
echo "Configuring Claude Desktop..."
|
||||
CONFIG_FILE="$HOME/Library/Application Support/Claude/claude_desktop_config.json"
|
||||
|
||||
# Create config directory if it doesn't exist
|
||||
mkdir -p "$(dirname "$CONFIG_FILE")"
|
||||
|
||||
# If config file doesn't exist, create it with initial structure
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
echo '{"mcpServers": {}}' > "$CONFIG_FILE"
|
||||
fi
|
||||
|
||||
# Add/update the basic-memory config using jq
|
||||
jq '.mcpServers."basic-memory" = {
|
||||
"command": "uvx",
|
||||
"args": ["basic-memory"]
|
||||
}' "$CONFIG_FILE" > "$CONFIG_FILE.tmp" && mv "$CONFIG_FILE.tmp" "$CONFIG_FILE"
|
||||
|
||||
echo "Installation complete! Basic Memory is now available in Claude Desktop."
|
||||
echo "Please restart Claude Desktop for changes to take effect."
|
||||
|
||||
echo -e "\nQuick Start:"
|
||||
echo "1. You can run sync directly using: uvx basic-memory sync"
|
||||
echo "2. Optionally, install globally with: uv pip install basic-memory"
|
||||
echo -e "\nBuilt with ♥️ by Basic Machines."
|
||||
@@ -1,3 +1,3 @@
|
||||
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
|
||||
|
||||
__version__ = "0.0.1"
|
||||
__version__ = "0.4.1"
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Generic single-database configuration.
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Alembic environment configuration."""
|
||||
|
||||
from logging.config import fileConfig
|
||||
|
||||
from sqlalchemy import engine_from_config
|
||||
from sqlalchemy import pool
|
||||
|
||||
from alembic import context
|
||||
|
||||
from basic_memory.models import Base
|
||||
from basic_memory.config import config as app_config
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
|
||||
# Set the SQLAlchemy URL from our app config
|
||||
sqlalchemy_url = f"sqlite:///{app_config.database_path}"
|
||||
config.set_main_option("sqlalchemy.url", sqlalchemy_url)
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
"""
|
||||
url = config.get_main_option("sqlalchemy.url")
|
||||
context.configure(
|
||||
url=url,
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
"""
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
|
||||
with connectable.connect() as connection:
|
||||
context.configure(connection=connection, target_metadata=target_metadata)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Functions for managing database migrations."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from loguru import logger
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
|
||||
|
||||
def get_alembic_config() -> Config: # pragma: no cover
|
||||
"""Get alembic config with correct paths."""
|
||||
migrations_path = Path(__file__).parent
|
||||
alembic_ini = migrations_path.parent.parent.parent / "alembic.ini"
|
||||
|
||||
config = Config(alembic_ini)
|
||||
config.set_main_option("script_location", str(migrations_path))
|
||||
return config
|
||||
|
||||
|
||||
async def reset_database(): # pragma: no cover
|
||||
"""Drop and recreate all tables."""
|
||||
logger.info("Resetting database...")
|
||||
config = get_alembic_config()
|
||||
|
||||
def _reset(cfg):
|
||||
command.downgrade(cfg, "base")
|
||||
command.upgrade(cfg, "head")
|
||||
|
||||
await asyncio.get_event_loop().run_in_executor(None, _reset, config)
|
||||
@@ -0,0 +1,26 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""initial schema
|
||||
|
||||
Revision ID: 3dae7c7b1564
|
||||
Revises:
|
||||
Create Date: 2025-02-12 21:23:00.336344
|
||||
|
||||
"""
|
||||
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = "3dae7c7b1564"
|
||||
down_revision: Union[str, None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"entity",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("title", sa.String(), nullable=False),
|
||||
sa.Column("entity_type", sa.String(), nullable=False),
|
||||
sa.Column("entity_metadata", sa.JSON(), nullable=True),
|
||||
sa.Column("content_type", sa.String(), nullable=False),
|
||||
sa.Column("permalink", sa.String(), nullable=False),
|
||||
sa.Column("file_path", sa.String(), nullable=False),
|
||||
sa.Column("checksum", sa.String(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(), nullable=False),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("permalink", name="uix_entity_permalink"),
|
||||
)
|
||||
op.create_index("ix_entity_created_at", "entity", ["created_at"], unique=False)
|
||||
op.create_index(op.f("ix_entity_file_path"), "entity", ["file_path"], unique=True)
|
||||
op.create_index(op.f("ix_entity_permalink"), "entity", ["permalink"], unique=True)
|
||||
op.create_index("ix_entity_title", "entity", ["title"], unique=False)
|
||||
op.create_index("ix_entity_type", "entity", ["entity_type"], unique=False)
|
||||
op.create_index("ix_entity_updated_at", "entity", ["updated_at"], unique=False)
|
||||
op.create_table(
|
||||
"observation",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("entity_id", sa.Integer(), nullable=False),
|
||||
sa.Column("content", sa.Text(), nullable=False),
|
||||
sa.Column("category", sa.String(), nullable=False),
|
||||
sa.Column("context", sa.Text(), nullable=True),
|
||||
sa.Column("tags", sa.JSON(), server_default="[]", nullable=True),
|
||||
sa.ForeignKeyConstraint(["entity_id"], ["entity.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
)
|
||||
op.create_index("ix_observation_category", "observation", ["category"], unique=False)
|
||||
op.create_index("ix_observation_entity_id", "observation", ["entity_id"], unique=False)
|
||||
op.create_table(
|
||||
"relation",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("from_id", sa.Integer(), nullable=False),
|
||||
sa.Column("to_id", sa.Integer(), nullable=True),
|
||||
sa.Column("to_name", sa.String(), nullable=False),
|
||||
sa.Column("relation_type", sa.String(), nullable=False),
|
||||
sa.Column("context", sa.Text(), nullable=True),
|
||||
sa.ForeignKeyConstraint(["from_id"], ["entity.id"], ondelete="CASCADE"),
|
||||
sa.ForeignKeyConstraint(["to_id"], ["entity.id"], ondelete="CASCADE"),
|
||||
sa.PrimaryKeyConstraint("id"),
|
||||
sa.UniqueConstraint("from_id", "to_id", "relation_type", name="uix_relation"),
|
||||
)
|
||||
op.create_index("ix_relation_from_id", "relation", ["from_id"], unique=False)
|
||||
op.create_index("ix_relation_to_id", "relation", ["to_id"], unique=False)
|
||||
op.create_index("ix_relation_type", "relation", ["relation_type"], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("ix_relation_type", table_name="relation")
|
||||
op.drop_index("ix_relation_to_id", table_name="relation")
|
||||
op.drop_index("ix_relation_from_id", table_name="relation")
|
||||
op.drop_table("relation")
|
||||
op.drop_index("ix_observation_entity_id", table_name="observation")
|
||||
op.drop_index("ix_observation_category", table_name="observation")
|
||||
op.drop_table("observation")
|
||||
op.drop_index("ix_entity_updated_at", table_name="entity")
|
||||
op.drop_index("ix_entity_type", table_name="entity")
|
||||
op.drop_index("ix_entity_title", table_name="entity")
|
||||
op.drop_index(op.f("ix_entity_permalink"), table_name="entity")
|
||||
op.drop_index(op.f("ix_entity_file_path"), table_name="entity")
|
||||
op.drop_index("ix_entity_created_at", table_name="entity")
|
||||
op.drop_table("entity")
|
||||
# ### end Alembic commands ###
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Basic Memory API module."""
|
||||
|
||||
from .app import app
|
||||
|
||||
__all__ = ["app"]
|
||||
__all__ = ["app"]
|
||||
|
||||
@@ -6,41 +6,22 @@ from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exception_handlers import http_exception_handler
|
||||
from loguru import logger
|
||||
|
||||
import basic_memory
|
||||
from basic_memory import db
|
||||
from basic_memory.config import config as app_config
|
||||
from basic_memory.api.routers import knowledge, search, memory, resource
|
||||
from basic_memory.config import config
|
||||
from basic_memory.services import DatabaseService
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
async def lifespan(app: FastAPI): # pragma: no cover
|
||||
"""Lifecycle manager for the FastAPI app."""
|
||||
logger.info("Starting Basic Memory API")
|
||||
|
||||
# check the db state
|
||||
await check_db(app)
|
||||
logger.info(f"Starting Basic Memory API {basic_memory.__version__}")
|
||||
await db.run_migrations(app_config)
|
||||
yield
|
||||
logger.info("Shutting down Basic Memory API")
|
||||
await db.shutdown_db()
|
||||
|
||||
|
||||
async def check_db(app: FastAPI):
|
||||
logger.info("Checking database state")
|
||||
|
||||
# Initialize DB management service
|
||||
db_service = DatabaseService(
|
||||
config=config,
|
||||
)
|
||||
|
||||
# Check and initialize DB if needed
|
||||
if not await db_service.check_db():
|
||||
raise RuntimeError("Database initialization failed")
|
||||
|
||||
# Clean up old backups on shutdown
|
||||
await db_service.cleanup_backups()
|
||||
|
||||
|
||||
|
||||
# Initialize FastAPI app
|
||||
app = FastAPI(
|
||||
title="Basic Memory API",
|
||||
@@ -57,7 +38,7 @@ app.include_router(resource.router)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def exception_handler(request, exc):
|
||||
async def exception_handler(request, exc): # pragma: no cover
|
||||
logger.exception(
|
||||
f"An unhandled exception occurred for request '{request.url}', exception: {exc}"
|
||||
)
|
||||
|
||||
@@ -17,7 +17,7 @@ from basic_memory.schemas import (
|
||||
DeleteEntitiesResponse,
|
||||
DeleteEntitiesRequest,
|
||||
)
|
||||
from basic_memory.schemas.base import PathId, Entity
|
||||
from basic_memory.schemas.base import Permalink, Entity
|
||||
from basic_memory.services.exceptions import EntityNotFoundError
|
||||
|
||||
router = APIRouter(prefix="/knowledge", tags=["knowledge"])
|
||||
@@ -27,10 +27,10 @@ router = APIRouter(prefix="/knowledge", tags=["knowledge"])
|
||||
|
||||
@router.post("/entities", response_model=EntityResponse)
|
||||
async def create_entity(
|
||||
data: Entity,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
data: Entity,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
) -> EntityResponse:
|
||||
"""Create an entity."""
|
||||
logger.info(f"request: create_entity with data={data}")
|
||||
@@ -47,12 +47,12 @@ async def create_entity(
|
||||
|
||||
@router.put("/entities/{permalink:path}", response_model=EntityResponse)
|
||||
async def create_or_update_entity(
|
||||
permalink: PathId,
|
||||
data: Entity,
|
||||
response: Response,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
permalink: Permalink,
|
||||
data: Entity,
|
||||
response: Response,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service: SearchServiceDep,
|
||||
) -> EntityResponse:
|
||||
"""Create or update an entity. If entity exists, it will be updated, otherwise created."""
|
||||
logger.info(f"request: create_or_update_entity with permalink={permalink}, data={data}")
|
||||
@@ -69,7 +69,9 @@ async def create_or_update_entity(
|
||||
await search_service.index_entity(entity, background_tasks=background_tasks)
|
||||
result = EntityResponse.model_validate(entity)
|
||||
|
||||
logger.info(f"response: create_or_update_entity with result={result}, status_code={response.status_code}")
|
||||
logger.info(
|
||||
f"response: create_or_update_entity with result={result}, status_code={response.status_code}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
@@ -78,8 +80,8 @@ async def create_or_update_entity(
|
||||
|
||||
@router.get("/entities/{permalink:path}", response_model=EntityResponse)
|
||||
async def get_entity(
|
||||
entity_service: EntityServiceDep,
|
||||
permalink: str,
|
||||
entity_service: EntityServiceDep,
|
||||
permalink: str,
|
||||
) -> EntityResponse:
|
||||
"""Get a specific entity by ID.
|
||||
|
||||
@@ -102,13 +104,13 @@ async def get_entity(
|
||||
|
||||
@router.get("/entities", response_model=EntityListResponse)
|
||||
async def get_entities(
|
||||
entity_service: EntityServiceDep,
|
||||
permalink: Annotated[list[str] | None, Query()] = None,
|
||||
entity_service: EntityServiceDep,
|
||||
permalink: Annotated[list[str] | None, Query()] = None,
|
||||
) -> EntityListResponse:
|
||||
"""Open specific entities"""
|
||||
logger.info(f"request: get_entities with permalinks={permalink}")
|
||||
|
||||
entities = await entity_service.get_entities_by_permalinks(permalink)
|
||||
entities = await entity_service.get_entities_by_permalinks(permalink) if permalink else []
|
||||
result = EntityListResponse(
|
||||
entities=[EntityResponse.model_validate(entity) for entity in entities]
|
||||
)
|
||||
@@ -122,11 +124,11 @@ async def get_entities(
|
||||
|
||||
@router.delete("/entities/{identifier:path}", response_model=DeleteEntitiesResponse)
|
||||
async def delete_entity(
|
||||
identifier: str,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
search_service=Depends(get_search_service),
|
||||
identifier: str,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
search_service=Depends(get_search_service),
|
||||
) -> DeleteEntitiesResponse:
|
||||
"""Delete a single entity and remove from search index."""
|
||||
logger.info(f"request: delete_entity with identifier={identifier}")
|
||||
@@ -149,10 +151,10 @@ async def delete_entity(
|
||||
|
||||
@router.post("/entities/delete", response_model=DeleteEntitiesResponse)
|
||||
async def delete_entities(
|
||||
data: DeleteEntitiesRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service=Depends(get_search_service),
|
||||
data: DeleteEntitiesRequest,
|
||||
background_tasks: BackgroundTasks,
|
||||
entity_service: EntityServiceDep,
|
||||
search_service=Depends(get_search_service),
|
||||
) -> DeleteEntitiesResponse:
|
||||
"""Delete entities and remove from search index."""
|
||||
logger.info(f"request: delete_entities with data={data}")
|
||||
|
||||
@@ -1,13 +1,11 @@
|
||||
"""Routes for memory:// URI operations."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, List, Annotated
|
||||
from typing import Annotated
|
||||
|
||||
from dateparser import parse
|
||||
from fastapi import APIRouter, Query
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.config import config
|
||||
from basic_memory.deps import ContextServiceDep, EntityRepositoryDep
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
@@ -17,7 +15,8 @@ from basic_memory.schemas.memory import (
|
||||
RelationSummary,
|
||||
EntitySummary,
|
||||
ObservationSummary,
|
||||
MemoryMetadata, normalize_memory_url,
|
||||
MemoryMetadata,
|
||||
normalize_memory_url,
|
||||
)
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
from basic_memory.services.context_service import ContextResultRow
|
||||
@@ -25,12 +24,14 @@ from basic_memory.services.context_service import ContextResultRow
|
||||
router = APIRouter(prefix="/memory", tags=["memory"])
|
||||
|
||||
|
||||
|
||||
async def to_graph_context(context, entity_repository: EntityRepository):
|
||||
# return results
|
||||
async def to_summary(item: SearchIndexRow | ContextResultRow):
|
||||
match item.type:
|
||||
case SearchItemType.ENTITY:
|
||||
assert item.title is not None
|
||||
assert item.created_at is not None
|
||||
|
||||
return EntitySummary(
|
||||
title=item.title,
|
||||
permalink=item.permalink,
|
||||
@@ -38,12 +39,18 @@ async def to_graph_context(context, entity_repository: EntityRepository):
|
||||
created_at=item.created_at,
|
||||
)
|
||||
case SearchItemType.OBSERVATION:
|
||||
assert item.category is not None
|
||||
assert item.content is not None
|
||||
|
||||
return ObservationSummary(
|
||||
category=item.category, content=item.content, permalink=item.permalink
|
||||
)
|
||||
case SearchItemType.RELATION:
|
||||
assert item.from_id is not None
|
||||
from_entity = await entity_repository.find_by_id(item.from_id)
|
||||
to_entity = await entity_repository.find_by_id(item.to_id)
|
||||
assert from_entity is not None
|
||||
|
||||
to_entity = await entity_repository.find_by_id(item.to_id) if item.to_id else None
|
||||
|
||||
return RelationSummary(
|
||||
permalink=item.permalink,
|
||||
@@ -51,6 +58,8 @@ async def to_graph_context(context, entity_repository: EntityRepository):
|
||||
from_id=from_entity.permalink,
|
||||
to_id=to_entity.permalink if to_entity else None,
|
||||
)
|
||||
case _: # pragma: no cover
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
primary_results = [await to_summary(r) for r in context["primary_results"]]
|
||||
related_results = [await to_summary(r) for r in context["related_results"]]
|
||||
@@ -61,7 +70,6 @@ async def to_graph_context(context, entity_repository: EntityRepository):
|
||||
)
|
||||
|
||||
|
||||
|
||||
@router.get("/recent", response_model=GraphContext)
|
||||
async def recent(
|
||||
context_service: ContextServiceDep,
|
||||
@@ -91,7 +99,8 @@ async def recent(
|
||||
return await to_graph_context(context, entity_repository=entity_repository)
|
||||
|
||||
|
||||
# get_memory_context needs to be declared last so other paths can match
|
||||
# get_memory_context needs to be declared last so other paths can match
|
||||
|
||||
|
||||
@router.get("/{uri:path}", response_model=GraphContext)
|
||||
async def get_memory_context(
|
||||
@@ -118,6 +127,3 @@ async def get_memory_context(
|
||||
memory_url, depth=depth, since=since, max_results=max_results
|
||||
)
|
||||
return await to_graph_context(context, entity_repository=entity_repository)
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,34 +1,28 @@
|
||||
"""Router for search operations."""
|
||||
|
||||
from dataclasses import asdict
|
||||
|
||||
from fastapi import APIRouter, Depends, BackgroundTasks
|
||||
from typing import List
|
||||
|
||||
from loguru import logger
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchResponse
|
||||
from basic_memory.deps import get_search_service
|
||||
|
||||
router = APIRouter(prefix="/search", tags=["search"])
|
||||
|
||||
|
||||
@router.post("/", response_model=SearchResponse)
|
||||
async def search(
|
||||
query: SearchQuery,
|
||||
search_service: SearchService = Depends(get_search_service)
|
||||
):
|
||||
async def search(query: SearchQuery, search_service: SearchService = Depends(get_search_service)):
|
||||
"""Search across all knowledge and documents."""
|
||||
results = await search_service.search(query)
|
||||
search_results = [SearchResult.model_validate(asdict(r)) for r in results]
|
||||
return SearchResponse(results=search_results)
|
||||
|
||||
|
||||
@router.post("/reindex")
|
||||
async def reindex(
|
||||
background_tasks: BackgroundTasks,
|
||||
search_service: SearchService = Depends(get_search_service)
|
||||
background_tasks: BackgroundTasks, search_service: SearchService = Depends(get_search_service)
|
||||
):
|
||||
"""Recreate and populate the search index."""
|
||||
await search_service.reindex_all(background_tasks=background_tasks)
|
||||
return {
|
||||
"status": "ok",
|
||||
"message": "Reindex initiated"
|
||||
}
|
||||
return {"status": "ok", "message": "Reindex initiated"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""CLI tools for basic-memory"""
|
||||
"""CLI tools for basic-memory"""
|
||||
|
||||
@@ -1,4 +1,20 @@
|
||||
import asyncio
|
||||
|
||||
import typer
|
||||
|
||||
app = typer.Typer()
|
||||
from basic_memory import db
|
||||
from basic_memory.config import config
|
||||
from basic_memory.utils import setup_logging
|
||||
|
||||
setup_logging(log_file=".basic-memory/basic-memory-cli.log") # pragma: no cover
|
||||
|
||||
asyncio.run(db.run_migrations(config))
|
||||
|
||||
app = typer.Typer(name="basic-memory")
|
||||
|
||||
import_app = typer.Typer()
|
||||
app.add_typer(import_app, name="import")
|
||||
|
||||
|
||||
claude_app = typer.Typer()
|
||||
import_app.add_typer(claude_app, name="claude")
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
"""Command module exports."""
|
||||
"""CLI commands for basic-memory."""
|
||||
|
||||
from . import init, status, sync, import_memory_json
|
||||
from . import status, sync, db, import_memory_json, mcp
|
||||
|
||||
__all__ = ["init", "status", "sync", "import_memory_json.py"]
|
||||
__all__ = ["status", "sync", "db", "import_memory_json", "mcp"]
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
"""Database management commands."""
|
||||
|
||||
import asyncio
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.alembic import migrations
|
||||
from basic_memory.cli.app import app
|
||||
|
||||
|
||||
@app.command()
|
||||
def reset(
|
||||
reindex: bool = typer.Option(False, "--reindex", help="Rebuild indices from filesystem"),
|
||||
): # pragma: no cover
|
||||
"""Reset database (drop all tables and recreate)."""
|
||||
if typer.confirm("This will delete all data. Are you sure?"):
|
||||
logger.info("Resetting database...")
|
||||
asyncio.run(migrations.reset_database())
|
||||
|
||||
if reindex:
|
||||
# Import and run sync
|
||||
from basic_memory.cli.commands.sync import sync
|
||||
|
||||
logger.info("Rebuilding search index from filesystem...")
|
||||
sync(watch=False) # pyright: ignore
|
||||
@@ -0,0 +1,255 @@
|
||||
"""Import command for ChatGPT conversations."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated, Set, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_timestamp(ts: float) -> str:
|
||||
"""Format Unix timestamp for display."""
|
||||
dt = datetime.fromtimestamp(ts)
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def get_message_content(message: Dict[str, Any]) -> str:
|
||||
"""Extract clean message content."""
|
||||
if not message or "content" not in message:
|
||||
return "" # pragma: no cover
|
||||
|
||||
content = message["content"]
|
||||
if content.get("content_type") == "text":
|
||||
return "\n".join(content.get("parts", []))
|
||||
elif content.get("content_type") == "code":
|
||||
return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
|
||||
return "" # pragma: no cover
|
||||
|
||||
|
||||
def traverse_messages(
|
||||
mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Traverse message tree and return messages in order."""
|
||||
messages = []
|
||||
node = mapping.get(root_id) if root_id else None
|
||||
|
||||
while node:
|
||||
if node["id"] not in seen and node.get("message"):
|
||||
seen.add(node["id"])
|
||||
messages.append(node["message"])
|
||||
|
||||
# Follow children
|
||||
children = node.get("children", [])
|
||||
for child_id in children:
|
||||
child_msgs = traverse_messages(mapping, child_id, seen)
|
||||
messages.extend(child_msgs)
|
||||
|
||||
break # Don't follow siblings
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def format_chat_markdown(
|
||||
title: str, mapping: Dict[str, Any], root_id: Optional[str], created_at: float, modified_at: float
|
||||
) -> str:
|
||||
"""Format chat as clean markdown."""
|
||||
|
||||
# Start with title
|
||||
lines = [f"# {title}\n"]
|
||||
|
||||
# Traverse message tree
|
||||
seen_msgs = set()
|
||||
messages = traverse_messages(mapping, root_id, seen_msgs)
|
||||
|
||||
# Format each message
|
||||
for msg in messages:
|
||||
# Skip hidden messages
|
||||
if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
|
||||
continue
|
||||
|
||||
# Get author and timestamp
|
||||
author = msg["author"]["role"].title()
|
||||
ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
|
||||
|
||||
# Add message header
|
||||
lines.append(f"### {author} ({ts})")
|
||||
|
||||
# Add message content
|
||||
content = get_message_content(msg)
|
||||
if content:
|
||||
lines.append(content)
|
||||
|
||||
# Add spacing
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_chat_content(folder: str, conversation: Dict[str, Any]) -> EntityMarkdown:
|
||||
"""Convert chat conversation to Basic Memory entity."""
|
||||
|
||||
# Extract timestamps
|
||||
created_at = conversation["create_time"]
|
||||
modified_at = conversation["update_time"]
|
||||
|
||||
root_id = None
|
||||
# Find root message
|
||||
for node_id, node in conversation["mapping"].items():
|
||||
if node.get("parent") is None:
|
||||
root_id = node_id
|
||||
break
|
||||
|
||||
# Generate permalink
|
||||
date_prefix = datetime.fromtimestamp(created_at).strftime("%Y%m%d")
|
||||
clean_title = clean_filename(conversation["title"])
|
||||
|
||||
# Format content
|
||||
content = format_chat_markdown(
|
||||
title=conversation["title"],
|
||||
mapping=conversation["mapping"],
|
||||
root_id=root_id,
|
||||
created_at=created_at,
|
||||
modified_at=modified_at,
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "conversation",
|
||||
"title": conversation["title"],
|
||||
"created": format_timestamp(created_at),
|
||||
"modified": format_timestamp(modified_at),
|
||||
"permalink": f"{folder}/{date_prefix}-{clean_title}",
|
||||
}
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_chatgpt_json(
|
||||
json_path: Path, folder: str, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import conversations from ChatGPT JSON format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading chat data...", total=None)
|
||||
|
||||
# Read conversations
|
||||
conversations = json.loads(json_path.read_text())
|
||||
progress.update(read_task, total=len(conversations))
|
||||
|
||||
# Process each conversation
|
||||
messages_imported = 0
|
||||
chats_imported = 0
|
||||
|
||||
for chat in conversations:
|
||||
# Convert to entity
|
||||
entity = format_chat_content(folder, chat)
|
||||
|
||||
# Write file
|
||||
file_path = config.home / f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
# logger.info(f"Writing file: {file_path.absolute()}")
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
|
||||
# Count messages
|
||||
msg_count = sum(
|
||||
1
|
||||
for node in chat["mapping"].values()
|
||||
if node.get("message")
|
||||
and not node.get("message", {})
|
||||
.get("metadata", {})
|
||||
.get("is_visually_hidden_from_conversation")
|
||||
)
|
||||
|
||||
chats_imported += 1
|
||||
messages_imported += msg_count
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"conversations": chats_imported, "messages": messages_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@import_app.command(name="chatgpt", help="Import conversations from ChatGPT JSON export.")
|
||||
def import_chatgpt(
|
||||
conversations_json: Annotated[
|
||||
Path, typer.Option(..., help="Path to ChatGPT conversations.json file")
|
||||
] = Path("conversations.json"),
|
||||
folder: Annotated[
|
||||
str, typer.Option(help="The folder to place the files in.")
|
||||
] = "conversations",
|
||||
):
|
||||
"""Import chat conversations from ChatGPT JSON format.
|
||||
|
||||
This command will:
|
||||
1. Read the complex tree structure of messages
|
||||
2. Convert them to linear markdown conversations
|
||||
3. Save as clean, readable markdown files
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if conversations_json:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_chatgpt_json(conversations_json, folder, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Import command for basic-memory CLI to import chat data from conversations2.json format."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import claude_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
# Remove invalid characters and convert spaces
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_timestamp(ts: str) -> str:
|
||||
"""Format ISO timestamp for display."""
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def format_chat_markdown(
|
||||
name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str
|
||||
) -> str:
|
||||
"""Format chat as clean markdown."""
|
||||
|
||||
# Start with frontmatter and title
|
||||
lines = [
|
||||
f"# {name}\n",
|
||||
]
|
||||
|
||||
# Add messages
|
||||
for msg in messages:
|
||||
# Format timestamp
|
||||
ts = format_timestamp(msg["created_at"])
|
||||
|
||||
# Add message header
|
||||
lines.append(f"### {msg['sender'].title()} ({ts})")
|
||||
|
||||
# Handle message content
|
||||
content = msg.get("text", "")
|
||||
if msg.get("content"):
|
||||
content = " ".join(c.get("text", "") for c in msg["content"])
|
||||
lines.append(content)
|
||||
|
||||
# Handle attachments
|
||||
attachments = msg.get("attachments", [])
|
||||
for attachment in attachments:
|
||||
if "file_name" in attachment:
|
||||
lines.append(f"\n**Attachment: {attachment['file_name']}**")
|
||||
if "extracted_content" in attachment:
|
||||
lines.append("```")
|
||||
lines.append(attachment["extracted_content"])
|
||||
lines.append("```")
|
||||
|
||||
# Add spacing between messages
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_chat_content(
|
||||
base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str
|
||||
) -> EntityMarkdown:
|
||||
"""Convert chat messages to Basic Memory entity format."""
|
||||
|
||||
# Generate permalink
|
||||
date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
|
||||
clean_title = clean_filename(name)
|
||||
permalink = f"{base_path}/{date_prefix}-{clean_title}"
|
||||
|
||||
# Format content
|
||||
content = format_chat_markdown(
|
||||
name=name,
|
||||
messages=messages,
|
||||
created_at=created_at,
|
||||
modified_at=modified_at,
|
||||
permalink=permalink,
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "conversation",
|
||||
"title": name,
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": permalink,
|
||||
}
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_conversations_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import chat data from conversations2.json format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading chat data...", total=None)
|
||||
|
||||
# Read chat data - handle array of arrays format
|
||||
data = json.loads(json_path.read_text())
|
||||
conversations = [chat for chat in data]
|
||||
progress.update(read_task, total=len(conversations))
|
||||
|
||||
# Process each conversation
|
||||
messages_imported = 0
|
||||
chats_imported = 0
|
||||
|
||||
for chat in conversations:
|
||||
# Convert to entity
|
||||
entity = format_chat_content(
|
||||
base_path=base_path,
|
||||
name=chat["name"],
|
||||
messages=chat["chat_messages"],
|
||||
created_at=chat["created_at"],
|
||||
modified_at=chat["updated_at"],
|
||||
)
|
||||
|
||||
# Write file
|
||||
file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md")
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
|
||||
chats_imported += 1
|
||||
messages_imported += len(chat["chat_messages"])
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"conversations": chats_imported, "messages": messages_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.")
|
||||
def import_claude(
|
||||
conversations_json: Annotated[
|
||||
Path, typer.Argument(..., help="Path to conversations.json file")
|
||||
] = Path("conversations.json"),
|
||||
folder: Annotated[
|
||||
str, typer.Option(help="The folder to place the files in.")
|
||||
] = "conversations",
|
||||
):
|
||||
"""Import chat conversations from conversations2.json format.
|
||||
|
||||
This command will:
|
||||
1. Read chat data and nested messages
|
||||
2. Create markdown files for each conversation
|
||||
3. Format content in clean, readable markdown
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_conversations_json(conversations_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Import command for basic-memory CLI to import project data from Claude.ai."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Annotated, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import claude_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_project_markdown(project: Dict[str, Any], doc: Dict[str, Any]) -> EntityMarkdown:
|
||||
"""Format a project document as a Basic Memory entity."""
|
||||
|
||||
# Extract timestamps
|
||||
created_at = doc.get("created_at") or project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean names for organization
|
||||
project_dir = clean_filename(project["name"])
|
||||
doc_file = clean_filename(doc["filename"])
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "project_doc",
|
||||
"title": doc["filename"],
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": f"{project_dir}/docs/{doc_file}",
|
||||
"project_name": project["name"],
|
||||
"project_uuid": project["uuid"],
|
||||
"doc_uuid": doc["uuid"],
|
||||
}
|
||||
),
|
||||
content=doc["content"],
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
def format_prompt_markdown(project: Dict[str, Any]) -> Optional[EntityMarkdown]:
|
||||
"""Format project prompt template as a Basic Memory entity."""
|
||||
|
||||
if not project.get("prompt_template"):
|
||||
return None
|
||||
|
||||
# Extract timestamps
|
||||
created_at = project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean project directory name
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "prompt_template",
|
||||
"title": f"Prompt Template: {project['name']}",
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": f"{project_dir}/prompt-template",
|
||||
"project_name": project["name"],
|
||||
"project_uuid": project["uuid"],
|
||||
}
|
||||
),
|
||||
content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}",
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_projects_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import project data from Claude.ai projects.json format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading project data...", total=None)
|
||||
|
||||
# Read project data
|
||||
data = json.loads(json_path.read_text())
|
||||
progress.update(read_task, total=len(data))
|
||||
|
||||
# Track import counts
|
||||
docs_imported = 0
|
||||
prompts_imported = 0
|
||||
|
||||
# Process each project
|
||||
for project in data:
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Create project directories
|
||||
docs_dir = base_path / project_dir / "docs"
|
||||
docs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Import prompt template if it exists
|
||||
if prompt_entity := format_prompt_markdown(project):
|
||||
file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
|
||||
await markdown_processor.write_file(file_path, prompt_entity)
|
||||
prompts_imported += 1
|
||||
|
||||
# Import project documents
|
||||
for doc in project.get("docs", []):
|
||||
entity = format_project_markdown(project, doc)
|
||||
file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
docs_imported += 1
|
||||
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"documents": docs_imported, "prompts": prompts_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@claude_app.command(name="projects", help="Import projects from Claude.ai.")
|
||||
def import_projects(
|
||||
projects_json: Annotated[Path, typer.Argument(..., help="Path to projects.json file")] = Path(
|
||||
"projects.json"
|
||||
),
|
||||
base_folder: Annotated[
|
||||
str, typer.Option(help="The base folder to place project files in.")
|
||||
] = "projects",
|
||||
):
|
||||
"""Import project data from Claude.ai.
|
||||
|
||||
This command will:
|
||||
1. Create a directory for each project
|
||||
2. Store docs in a docs/ subdirectory
|
||||
3. Place prompt template in project root
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if projects_json:
|
||||
if not projects_json.exists():
|
||||
typer.echo(f"Error: File not found: {projects_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / base_folder if base_folder else config.home
|
||||
console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_projects_json(projects_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['documents']} project documents\n"
|
||||
f"Imported {results['prompts']} prompt templates",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -3,7 +3,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
@@ -11,21 +11,23 @@ from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation, Relation
|
||||
|
||||
|
||||
console = Console()
|
||||
|
||||
async def process_memory_json(json_path: Path, base_path: Path,markdown_processor: MarkdownProcessor):
|
||||
|
||||
async def process_memory_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
):
|
||||
"""Import entities from memory.json using markdown processor."""
|
||||
|
||||
|
||||
# First pass - collect all relations by source entity
|
||||
entity_relations: Dict[str, List[Relation]] = {}
|
||||
entities: Dict[str, Dict[str, Any]] = {}
|
||||
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
@@ -34,12 +36,12 @@ async def process_memory_json(json_path: Path, base_path: Path,markdown_processo
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading memory.json...", total=None)
|
||||
|
||||
|
||||
# First pass - collect entities and relations
|
||||
with open(json_path) as f:
|
||||
lines = f.readlines()
|
||||
progress.update(read_task, total=len(lines))
|
||||
|
||||
|
||||
for line in lines:
|
||||
data = json.loads(line)
|
||||
if data["type"] == "entity":
|
||||
@@ -52,14 +54,14 @@ async def process_memory_json(json_path: Path, base_path: Path,markdown_processo
|
||||
entity_relations[source].append(
|
||||
Relation(
|
||||
type=data.get("relationType") or data.get("relation_type"),
|
||||
target=data.get("to") or data.get("to_id")
|
||||
target=data.get("to") or data.get("to_id"),
|
||||
)
|
||||
)
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
# Second pass - create and write entities
|
||||
write_task = progress.add_task("Creating entities...", total=len(entities))
|
||||
|
||||
|
||||
entities_created = 0
|
||||
for name, entity_data in entities.items():
|
||||
entity = EntityMarkdown(
|
||||
@@ -67,26 +69,25 @@ async def process_memory_json(json_path: Path, base_path: Path,markdown_processo
|
||||
metadata={
|
||||
"type": entity_data["entityType"],
|
||||
"title": name,
|
||||
"permalink": f"{entity_data['entityType']}/{name}"
|
||||
"permalink": f"{entity_data['entityType']}/{name}",
|
||||
}
|
||||
),
|
||||
content=f"# {name}\n",
|
||||
observations=[
|
||||
Observation(content=obs)
|
||||
for obs in entity_data["observations"]
|
||||
],
|
||||
relations=entity_relations.get(name, []) # Add any relations where this entity is the source
|
||||
observations=[Observation(content=obs) for obs in entity_data["observations"]],
|
||||
relations=entity_relations.get(
|
||||
name, []
|
||||
), # Add any relations where this entity is the source
|
||||
)
|
||||
|
||||
|
||||
# Let markdown processor handle writing
|
||||
file_path = base_path / f"{entity_data['entityType']}/{name}.md"
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
entities_created += 1
|
||||
progress.update(write_task, advance=1)
|
||||
|
||||
|
||||
return {
|
||||
"entities": entities_created,
|
||||
"relations": sum(len(rels) for rels in entity_relations.values())
|
||||
"relations": sum(len(rels) for rels in entity_relations.values()),
|
||||
}
|
||||
|
||||
|
||||
@@ -96,44 +97,48 @@ async def get_markdown_processor() -> MarkdownProcessor:
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@app.command()
|
||||
def import_json(
|
||||
json_path: Path = typer.Argument(..., help="Path to memory.json file to import"),
|
||||
@import_app.command()
|
||||
def memory_json(
|
||||
json_path: Annotated[Path, typer.Argument(..., help="Path to memory.json file")] = Path(
|
||||
"memory.json"
|
||||
),
|
||||
):
|
||||
"""Import entities and relations from a memory.json file.
|
||||
|
||||
|
||||
This command will:
|
||||
1. Read entities and relations from the JSON file
|
||||
2. Create markdown files for each entity
|
||||
3. Include outgoing relations in each entity's markdown
|
||||
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
|
||||
if not json_path.exists():
|
||||
typer.echo(f"Error: File not found: {json_path}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
|
||||
try:
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
|
||||
# Process the file
|
||||
base_path = config.home
|
||||
console.print(f"\nImporting from {json_path}...writing to {base_path}")
|
||||
results = asyncio.run(process_memory_json(json_path, base_path, markdown_processor))
|
||||
|
||||
|
||||
# Show results
|
||||
console.print(Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Created {results['entities']} entities\n"
|
||||
f"Added {results['relations']} relations",
|
||||
expand=False
|
||||
))
|
||||
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Created {results['entities']} entities\n"
|
||||
f"Added {results['relations']} relations",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Import failed")
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
"""Initialize command for basic-memory CLI."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.db import engine_session_factory, DatabaseType
|
||||
from basic_memory.config import config
|
||||
|
||||
|
||||
async def _init(force: bool = False):
|
||||
"""Initialize the database."""
|
||||
db_path = config.database_path
|
||||
|
||||
if db_path.exists() and not force:
|
||||
typer.echo(f"Database already exists at {db_path}. Use --force to reinitialize.")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Create data directory if needed
|
||||
db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
async with engine_session_factory(db_path, db_type=DatabaseType.FILESYSTEM, init=True):
|
||||
typer.echo(f"Initialized database at {db_path}")
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing database: {e}")
|
||||
typer.echo(f"Error initializing database: {e}")
|
||||
raise typer.Exit(1)
|
||||
|
||||
@app.command()
|
||||
def init(
|
||||
force: bool = typer.Option(False, "--force", "-f", help="Force reinitialization if database exists")
|
||||
):
|
||||
"""Initialize a new basic-memory database."""
|
||||
asyncio.run(_init(force))
|
||||
@@ -0,0 +1,20 @@
|
||||
"""MCP server command."""
|
||||
|
||||
from loguru import logger
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.config import config
|
||||
|
||||
# Import mcp instance
|
||||
from basic_memory.mcp.server import mcp as mcp_server # pragma: no cover
|
||||
|
||||
# Import mcp tools to register them
|
||||
import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
|
||||
|
||||
|
||||
@app.command()
|
||||
def mcp(): # pragma: no cover
|
||||
"""Run the MCP server for Claude Desktop integration."""
|
||||
home_dir = config.home
|
||||
logger.info(f"Starting Basic Memory MCP server {basic_memory.__version__}")
|
||||
logger.info(f"Home directory: {home_dir}")
|
||||
mcp_server.run()
|
||||
@@ -21,18 +21,20 @@ from basic_memory.sync.utils import SyncReport
|
||||
console = Console()
|
||||
|
||||
|
||||
async def get_file_change_scanner(db_type=DatabaseType.FILESYSTEM) -> FileChangeScanner:
|
||||
async def get_file_change_scanner(
|
||||
db_type=DatabaseType.FILESYSTEM,
|
||||
) -> FileChangeScanner: # pragma: no cover
|
||||
"""Get sync service instance."""
|
||||
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
|
||||
engine,
|
||||
session_maker,
|
||||
):
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
return file_change_scanner
|
||||
_, session_maker = await db.get_or_create_db(db_path=config.database_path, db_type=db_type)
|
||||
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
return file_change_scanner
|
||||
|
||||
|
||||
def add_files_to_tree(tree: Tree, paths: Set[str], style: str, checksums: Dict[str, str] = None):
|
||||
def add_files_to_tree(
|
||||
tree: Tree, paths: Set[str], style: str, checksums: Dict[str, str] | None = None
|
||||
):
|
||||
"""Add files to tree, grouped by directory."""
|
||||
# Group by directory
|
||||
by_dir = {}
|
||||
@@ -126,7 +128,8 @@ def display_changes(title: str, changes: SyncReport, verbose: bool = False):
|
||||
by_dir = group_changes_by_directory(changes)
|
||||
for dir_name, counts in sorted(by_dir.items()):
|
||||
summary = build_directory_summary(counts)
|
||||
tree.add(f"[bold]{dir_name}/[/bold] {summary}")
|
||||
if summary: # Only show directories with changes
|
||||
tree.add(f"[bold]{dir_name}/[/bold] {summary}")
|
||||
|
||||
console.print(Panel(tree, expand=False))
|
||||
|
||||
@@ -145,8 +148,7 @@ def status(
|
||||
"""Show sync status between files and database."""
|
||||
try:
|
||||
sync_service = asyncio.run(get_file_change_scanner())
|
||||
asyncio.run(run_status(sync_service, verbose))
|
||||
asyncio.run(run_status(sync_service, verbose)) # pragma: no cover
|
||||
except Exception as e:
|
||||
logger.exception(f"Error checking status: {e}")
|
||||
typer.echo(f"Error checking status: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
raise typer.Exit(code=1) # pragma: no cover
|
||||
|
||||
@@ -9,15 +9,11 @@ from typing import List, Dict
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.padding import Padding
|
||||
from rich.panel import Panel
|
||||
from rich.text import Text
|
||||
from rich.tree import Tree
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.repository import (
|
||||
@@ -42,50 +38,50 @@ class ValidationIssue:
|
||||
error: str
|
||||
|
||||
|
||||
async def get_sync_service(db_type=DatabaseType.FILESYSTEM):
|
||||
async def get_sync_service(): # pragma: no cover
|
||||
"""Get sync service instance with all dependencies."""
|
||||
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
|
||||
engine,
|
||||
session_maker,
|
||||
):
|
||||
entity_parser = EntityParser(config.home)
|
||||
markdown_processor = MarkdownProcessor(entity_parser)
|
||||
file_service = FileService(config.home, markdown_processor)
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=config.database_path, db_type=db.DatabaseType.FILESYSTEM
|
||||
)
|
||||
|
||||
# Initialize repositories
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
observation_repository = ObservationRepository(session_maker)
|
||||
relation_repository = RelationRepository(session_maker)
|
||||
search_repository = SearchRepository(session_maker)
|
||||
entity_parser = EntityParser(config.home)
|
||||
markdown_processor = MarkdownProcessor(entity_parser)
|
||||
file_service = FileService(config.home, markdown_processor)
|
||||
|
||||
# Initialize services
|
||||
search_service = SearchService(search_repository, entity_repository, file_service)
|
||||
link_resolver = LinkResolver(entity_repository, search_service)
|
||||
# Initialize repositories
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
observation_repository = ObservationRepository(session_maker)
|
||||
relation_repository = RelationRepository(session_maker)
|
||||
search_repository = SearchRepository(session_maker)
|
||||
|
||||
# Initialize scanner
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
# Initialize services
|
||||
search_service = SearchService(search_repository, entity_repository, file_service)
|
||||
link_resolver = LinkResolver(entity_repository, search_service)
|
||||
|
||||
# Initialize services
|
||||
entity_service = EntityService(
|
||||
entity_parser,
|
||||
entity_repository,
|
||||
observation_repository,
|
||||
relation_repository,
|
||||
file_service,
|
||||
link_resolver,
|
||||
)
|
||||
# Initialize scanner
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
|
||||
# Create sync service
|
||||
sync_service = SyncService(
|
||||
scanner=file_change_scanner,
|
||||
entity_service=entity_service,
|
||||
entity_parser=entity_parser,
|
||||
entity_repository=entity_repository,
|
||||
relation_repository=relation_repository,
|
||||
search_service=search_service,
|
||||
)
|
||||
# Initialize services
|
||||
entity_service = EntityService(
|
||||
entity_parser,
|
||||
entity_repository,
|
||||
observation_repository,
|
||||
relation_repository,
|
||||
file_service,
|
||||
link_resolver,
|
||||
)
|
||||
|
||||
return sync_service
|
||||
# Create sync service
|
||||
sync_service = SyncService(
|
||||
scanner=file_change_scanner,
|
||||
entity_service=entity_service,
|
||||
entity_parser=entity_parser,
|
||||
entity_repository=entity_repository,
|
||||
relation_repository=relation_repository,
|
||||
search_service=search_service,
|
||||
)
|
||||
|
||||
return sync_service
|
||||
|
||||
|
||||
def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]:
|
||||
@@ -97,53 +93,6 @@ def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[V
|
||||
return dict(grouped)
|
||||
|
||||
|
||||
def display_validation_errors(issues: List[ValidationIssue]):
|
||||
"""Display validation errors in a rich, organized format."""
|
||||
# Create header
|
||||
console.print()
|
||||
console.print(
|
||||
Panel("[red bold]Error:[/red bold] Invalid frontmatter in knowledge files", expand=False)
|
||||
)
|
||||
console.print()
|
||||
|
||||
# Group issues by directory
|
||||
grouped_issues = group_issues_by_directory(issues)
|
||||
|
||||
# Create tree structure
|
||||
tree = Tree("Knowledge Files")
|
||||
for dir_name, dir_issues in sorted(grouped_issues.items()):
|
||||
# Create branch for directory
|
||||
branch = tree.add(
|
||||
f"[bold blue]{dir_name}/[/bold blue] ([yellow]{len(dir_issues)} files[/yellow])"
|
||||
)
|
||||
|
||||
# Add each file issue
|
||||
for issue in sorted(dir_issues, key=lambda x: x.file_path):
|
||||
file_name = Path(issue.file_path).name
|
||||
branch.add(
|
||||
Text.assemble(("└─ ", "dim"), (file_name, "yellow"), ": ", (issue.error, "red"))
|
||||
)
|
||||
|
||||
# Display tree
|
||||
console.print(Padding(tree, (1, 2)))
|
||||
|
||||
# Add help text
|
||||
console.print()
|
||||
console.print(
|
||||
Panel(
|
||||
Text.assemble(
|
||||
("To fix:", "bold"),
|
||||
"\n1. Add required frontmatter fields to each file",
|
||||
"\n2. Run ",
|
||||
("basic-memory sync", "bold cyan"),
|
||||
" again",
|
||||
),
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
console.print()
|
||||
|
||||
|
||||
def display_sync_summary(knowledge: SyncReport):
|
||||
"""Display a one-line summary of sync changes."""
|
||||
total_changes = knowledge.total_changes
|
||||
@@ -212,10 +161,10 @@ async def run_sync(verbose: bool = False, watch: bool = False):
|
||||
watch_service = WatchService(
|
||||
sync_service=sync_service,
|
||||
file_service=sync_service.entity_service.file_service,
|
||||
config=config
|
||||
config=config,
|
||||
)
|
||||
await watch_service.handle_changes(config.home)
|
||||
await watch_service.run()
|
||||
await watch_service.run() # pragma: no cover
|
||||
else:
|
||||
# one time sync
|
||||
knowledge_changes = await sync_service.sync(config.home)
|
||||
@@ -223,7 +172,7 @@ async def run_sync(verbose: bool = False, watch: bool = False):
|
||||
if verbose:
|
||||
display_detailed_sync_results(knowledge_changes)
|
||||
else:
|
||||
display_sync_summary(knowledge_changes)
|
||||
display_sync_summary(knowledge_changes) # pragma: no cover
|
||||
|
||||
|
||||
@app.command()
|
||||
@@ -246,7 +195,7 @@ def sync(
|
||||
# Run sync
|
||||
asyncio.run(run_sync(verbose=verbose, watch=watch))
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
if not isinstance(e, typer.Exit):
|
||||
logger.exception("Sync failed")
|
||||
typer.echo(f"Error during sync: {e}", err=True)
|
||||
|
||||
@@ -1,48 +1,23 @@
|
||||
"""Main CLI entry point for basic-memory."""
|
||||
import sys
|
||||
"""Main CLI entry point for basic-memory.""" # pragma: no cover
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands.init import init
|
||||
from basic_memory.cli.app import app # pragma: no cover
|
||||
from basic_memory.utils import setup_logging # pragma: no cover
|
||||
|
||||
# Register commands
|
||||
from basic_memory.cli.commands import init, status, sync
|
||||
__all__ = ["init", "status", "sync"]
|
||||
from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
|
||||
status,
|
||||
sync,
|
||||
db,
|
||||
import_memory_json,
|
||||
mcp,
|
||||
import_claude_conversations,
|
||||
import_claude_projects,
|
||||
import_chatgpt,
|
||||
)
|
||||
|
||||
from basic_memory.config import config
|
||||
|
||||
|
||||
def setup_logging(home_dir: str = config.home, log_file: str = ".basic-memory/basic-memory-tools.log"):
|
||||
"""Configure logging for the application."""
|
||||
|
||||
# Remove default handler and any existing handlers
|
||||
logger.remove()
|
||||
|
||||
# Add file handler for debug level logs
|
||||
log = f"{home_dir}/{log_file}"
|
||||
logger.add(
|
||||
log,
|
||||
level="DEBUG",
|
||||
rotation="100 MB",
|
||||
retention="10 days",
|
||||
backtrace=True,
|
||||
diagnose=True,
|
||||
enqueue=True,
|
||||
colorize=False,
|
||||
)
|
||||
|
||||
# Add stderr handler for warnings and errors only
|
||||
logger.add(
|
||||
sys.stderr,
|
||||
level="WARNING",
|
||||
backtrace=True,
|
||||
diagnose=True
|
||||
)
|
||||
|
||||
# Set up logging when module is imported
|
||||
setup_logging()
|
||||
setup_logging(log_file=".basic-memory/basic-memory-cli.log") # pragma: no cover
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
app()
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from loguru import logger
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
@@ -27,6 +26,8 @@ class ProjectConfig(BaseSettings):
|
||||
default=500, description="Milliseconds to wait after changes before syncing", gt=0
|
||||
)
|
||||
|
||||
log_level: str = "INFO"
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="BASIC_MEMORY_",
|
||||
extra="ignore",
|
||||
@@ -45,7 +46,7 @@ class ProjectConfig(BaseSettings):
|
||||
|
||||
@field_validator("home")
|
||||
@classmethod
|
||||
def ensure_path_exists(cls, v: Path) -> Path:
|
||||
def ensure_path_exists(cls, v: Path) -> Path: # pragma: no cover
|
||||
"""Ensure project path exists."""
|
||||
if not v.exists():
|
||||
v.mkdir(parents=True)
|
||||
@@ -54,4 +55,3 @@ class ProjectConfig(BaseSettings):
|
||||
|
||||
# Load project config
|
||||
config = ProjectConfig()
|
||||
logger.info(f"project config home: {config.home}")
|
||||
|
||||
+38
-42
@@ -4,6 +4,10 @@ from enum import Enum, auto
|
||||
from pathlib import Path
|
||||
from typing import AsyncGenerator, Optional
|
||||
|
||||
from basic_memory.config import ProjectConfig
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
@@ -14,7 +18,7 @@ from sqlalchemy.ext.asyncio import (
|
||||
async_scoped_session,
|
||||
)
|
||||
|
||||
from basic_memory.models import Base, SCHEMA_VERSION
|
||||
from basic_memory.repository.search_repository import SearchRepository
|
||||
|
||||
# Module level state
|
||||
_engine: Optional[AsyncEngine] = None
|
||||
@@ -34,7 +38,7 @@ class DatabaseType(Enum):
|
||||
logger.info("Using in-memory SQLite database")
|
||||
return "sqlite+aiosqlite://"
|
||||
|
||||
return f"sqlite+aiosqlite:///{db_path}"
|
||||
return f"sqlite+aiosqlite:///{db_path}" # pragma: no cover
|
||||
|
||||
|
||||
def get_scoped_session_factory(
|
||||
@@ -68,37 +72,10 @@ async def scoped_session(
|
||||
await factory.remove()
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
"""Initialize database with required tables."""
|
||||
|
||||
logger.info("Initializing database...")
|
||||
|
||||
async with scoped_session(_session_maker) as session:
|
||||
await session.execute(text("PRAGMA foreign_keys=ON"))
|
||||
conn = await session.connection()
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
await session.commit()
|
||||
|
||||
async def drop_db():
|
||||
"""Drop all database tables."""
|
||||
global _engine, _session_maker
|
||||
|
||||
logger.info("Dropping tables...")
|
||||
async with scoped_session(_session_maker) as session:
|
||||
conn = await session.connection()
|
||||
await conn.run_sync(Base.metadata.drop_all)
|
||||
await session.commit()
|
||||
|
||||
# reset global engine and session_maker
|
||||
_engine = None
|
||||
_session_maker = None
|
||||
|
||||
|
||||
async def get_or_create_db(
|
||||
db_path: Path,
|
||||
db_type: DatabaseType = DatabaseType.FILESYSTEM,
|
||||
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
|
||||
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]: # pragma: no cover
|
||||
"""Get or create database engine and session maker."""
|
||||
global _engine, _session_maker
|
||||
|
||||
@@ -108,13 +85,12 @@ async def get_or_create_db(
|
||||
_engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
|
||||
_session_maker = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Initialize database
|
||||
await init_db()
|
||||
|
||||
assert _engine is not None # for type checker
|
||||
assert _session_maker is not None # for type checker
|
||||
return _engine, _session_maker
|
||||
|
||||
|
||||
async def shutdown_db():
|
||||
async def shutdown_db() -> None: # pragma: no cover
|
||||
"""Clean up database connections."""
|
||||
global _engine, _session_maker
|
||||
|
||||
@@ -124,12 +100,10 @@ async def shutdown_db():
|
||||
_session_maker = None
|
||||
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def engine_session_factory(
|
||||
db_path: Path,
|
||||
db_type: DatabaseType = DatabaseType.MEMORY,
|
||||
init: bool = True,
|
||||
) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
|
||||
"""Create engine and session factory.
|
||||
|
||||
@@ -138,17 +112,39 @@ async def engine_session_factory(
|
||||
"""
|
||||
|
||||
global _engine, _session_maker
|
||||
|
||||
|
||||
db_url = DatabaseType.get_db_url(db_path, db_type)
|
||||
logger.debug(f"Creating engine for db_url: {db_url}")
|
||||
|
||||
|
||||
_engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
|
||||
try:
|
||||
_session_maker = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
if init:
|
||||
await init_db()
|
||||
|
||||
assert _engine is not None # for type checker
|
||||
assert _session_maker is not None # for type checker
|
||||
yield _engine, _session_maker
|
||||
finally:
|
||||
await _engine.dispose()
|
||||
if _engine:
|
||||
await _engine.dispose()
|
||||
_engine = None
|
||||
_session_maker = None
|
||||
|
||||
|
||||
async def run_migrations(app_config: ProjectConfig, database_type=DatabaseType.FILESYSTEM):
|
||||
"""Run any pending alembic migrations."""
|
||||
logger.info("Running database migrations...")
|
||||
try:
|
||||
# Get the absolute path to the alembic directory relative to this file
|
||||
alembic_dir = Path(__file__).parent / "alembic"
|
||||
config = Config()
|
||||
config.set_main_option("script_location", str(alembic_dir))
|
||||
config.set_main_option("sqlalchemy.url", "driver://user:pass@localhost/dbname")
|
||||
|
||||
command.upgrade(config, "head")
|
||||
logger.info("Migrations completed successfully")
|
||||
|
||||
_, session_maker = await get_or_create_db(app_config.database_path, database_type)
|
||||
await SearchRepository(session_maker).init_search_index()
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error running migrations: {e}")
|
||||
raise
|
||||
@@ -29,11 +29,11 @@ from basic_memory.services.search_service import SearchService
|
||||
## project
|
||||
|
||||
|
||||
def get_project_config() -> ProjectConfig:
|
||||
def get_project_config() -> ProjectConfig: # pragma: no cover
|
||||
return config
|
||||
|
||||
|
||||
ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]
|
||||
ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)] # pragma: no cover
|
||||
|
||||
|
||||
## sqlalchemy
|
||||
@@ -41,7 +41,7 @@ ProjectConfigDep = Annotated[ProjectConfig, Depends(get_project_config)]
|
||||
|
||||
async def get_engine_factory(
|
||||
project_config: ProjectConfigDep,
|
||||
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]:
|
||||
) -> tuple[AsyncEngine, async_sessionmaker[AsyncSession]]: # pragma: no cover
|
||||
"""Get engine and session maker."""
|
||||
return await db.get_or_create_db(project_config.database_path)
|
||||
|
||||
@@ -129,7 +129,6 @@ async def get_file_service(
|
||||
FileServiceDep = Annotated[FileService, Depends(get_file_service)]
|
||||
|
||||
|
||||
|
||||
async def get_entity_service(
|
||||
entity_repository: EntityRepositoryDep,
|
||||
observation_repository: ObservationRepositoryDep,
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
"""Utilities for file operations."""
|
||||
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Tuple
|
||||
from typing import Dict, Any
|
||||
|
||||
import yaml
|
||||
from loguru import logger
|
||||
@@ -9,35 +10,38 @@ from loguru import logger
|
||||
|
||||
class FileError(Exception):
|
||||
"""Base exception for file operations."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class FileWriteError(FileError):
|
||||
"""Raised when file operations fail."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class ParseError(FileError):
|
||||
"""Raised when parsing file content fails."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
async def compute_checksum(content: str) -> str:
|
||||
"""
|
||||
Compute SHA-256 checksum of content.
|
||||
|
||||
|
||||
Args:
|
||||
content: Text content to hash
|
||||
|
||||
|
||||
Returns:
|
||||
SHA-256 hex digest
|
||||
|
||||
|
||||
Raises:
|
||||
FileError: If checksum computation fails
|
||||
"""
|
||||
try:
|
||||
return hashlib.sha256(content.encode()).hexdigest()
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to compute checksum: {e}")
|
||||
raise FileError(f"Failed to compute checksum: {e}")
|
||||
|
||||
@@ -45,16 +49,16 @@ async def compute_checksum(content: str) -> str:
|
||||
async def ensure_directory(path: Path) -> None:
|
||||
"""
|
||||
Ensure directory exists, creating if necessary.
|
||||
|
||||
|
||||
Args:
|
||||
path: Directory path to ensure
|
||||
|
||||
|
||||
Raises:
|
||||
FileWriteError: If directory creation fails
|
||||
"""
|
||||
try:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to create directory: {path}: {e}")
|
||||
raise FileWriteError(f"Failed to create directory {path}: {e}")
|
||||
|
||||
@@ -62,22 +66,20 @@ async def ensure_directory(path: Path) -> None:
|
||||
async def write_file_atomic(path: Path, content: str) -> None:
|
||||
"""
|
||||
Write file with atomic operation using temporary file.
|
||||
|
||||
|
||||
Args:
|
||||
path: Target file path
|
||||
content: Content to write
|
||||
|
||||
|
||||
Raises:
|
||||
FileWriteError: If write operation fails
|
||||
"""
|
||||
temp_path = path.with_suffix(".tmp")
|
||||
try:
|
||||
temp_path.write_text(content)
|
||||
|
||||
# TODO check for path.exists()
|
||||
temp_path.replace(path)
|
||||
logger.debug(f"wrote file: {path}")
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
temp_path.unlink(missing_ok=True)
|
||||
logger.error(f"Failed to write file: {path}: {e}")
|
||||
raise FileWriteError(f"Failed to write file {path}: {e}")
|
||||
@@ -85,16 +87,19 @@ async def write_file_atomic(path: Path, content: str) -> None:
|
||||
|
||||
def has_frontmatter(content: str) -> bool:
|
||||
"""
|
||||
Check if content contains YAML frontmatter.
|
||||
Check if content contains valid YAML frontmatter.
|
||||
|
||||
Args:
|
||||
content: Content to check
|
||||
|
||||
Returns:
|
||||
True if content has frontmatter delimiter (---), False otherwise
|
||||
True if content has valid frontmatter markers (---), False otherwise
|
||||
"""
|
||||
content = content.strip()
|
||||
return content.startswith("---") and "---" in content[3:]
|
||||
if not content.startswith("---"):
|
||||
return False
|
||||
|
||||
return "---" in content[3:]
|
||||
|
||||
|
||||
def parse_frontmatter(content: str) -> Dict[str, Any]:
|
||||
@@ -111,7 +116,7 @@ def parse_frontmatter(content: str) -> Dict[str, Any]:
|
||||
ParseError: If frontmatter is invalid or parsing fails
|
||||
"""
|
||||
try:
|
||||
if not has_frontmatter(content):
|
||||
if not content.strip().startswith("---"):
|
||||
raise ParseError("Content has no frontmatter")
|
||||
|
||||
# Split on first two occurrences of ---
|
||||
@@ -132,7 +137,7 @@ def parse_frontmatter(content: str) -> Dict[str, Any]:
|
||||
except yaml.YAMLError as e:
|
||||
raise ParseError(f"Invalid YAML in frontmatter: {e}")
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
if not isinstance(e, ParseError):
|
||||
logger.error(f"Failed to parse frontmatter: {e}")
|
||||
raise ParseError(f"Failed to parse frontmatter: {e}")
|
||||
@@ -147,27 +152,23 @@ def remove_frontmatter(content: str) -> str:
|
||||
content: Content with frontmatter
|
||||
|
||||
Returns:
|
||||
Content with frontmatter removed
|
||||
Content with frontmatter removed, or original content if no frontmatter
|
||||
|
||||
Raises:
|
||||
ParseError: If frontmatter format is invalid
|
||||
ParseError: If content starts with frontmatter marker but is malformed
|
||||
"""
|
||||
try:
|
||||
if not has_frontmatter(content):
|
||||
return content.strip()
|
||||
content = content.strip()
|
||||
|
||||
# Split on first two occurrences of ---
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("Invalid frontmatter format")
|
||||
# Return as-is if no frontmatter marker
|
||||
if not content.startswith("---"):
|
||||
return content
|
||||
|
||||
return parts[2].strip()
|
||||
# Split on first two occurrences of ---
|
||||
parts = content.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
raise ParseError("Invalid frontmatter format")
|
||||
|
||||
except Exception as e:
|
||||
if not isinstance(e, ParseError):
|
||||
logger.error(f"Failed to remove frontmatter: {e}")
|
||||
raise ParseError(f"Failed to remove frontmatter: {e}")
|
||||
raise
|
||||
return parts[2].strip()
|
||||
|
||||
|
||||
async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
|
||||
@@ -208,6 +209,6 @@ async def update_frontmatter(path: Path, updates: Dict[str, Any]) -> str:
|
||||
await write_file_atomic(path, final_content)
|
||||
return await compute_checksum(final_content)
|
||||
|
||||
except Exception as e:
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Failed to update frontmatter in {path}: {e}")
|
||||
raise FileError(f"Failed to update frontmatter: {e}")
|
||||
raise FileError(f"Failed to update frontmatter: {e}")
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
Uses markdown-it with plugins to parse structured data from markdown content.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
@@ -21,11 +22,12 @@ from basic_memory.markdown.schemas import (
|
||||
|
||||
md = MarkdownIt().use(observation_plugin).use(relation_plugin)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EntityContent:
|
||||
content: str
|
||||
observations: list[Observation] = field(default_factory=list)
|
||||
relations: list[Relation] = field(default_factory=list)
|
||||
content: str
|
||||
observations: list[Observation] = field(default_factory=list)
|
||||
relations: list[Relation] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse(content: str) -> EntityContent:
|
||||
@@ -53,13 +55,13 @@ def parse(content: str) -> EntityContent:
|
||||
relations=relations,
|
||||
)
|
||||
|
||||
|
||||
def parse_tags(tags: Any) -> list[str]:
|
||||
"""Parse tags into list of strings."""
|
||||
if isinstance(tags, str):
|
||||
return [t.strip() for t in tags.split(",") if t.strip()]
|
||||
if isinstance(tags, (list, tuple)):
|
||||
return [str(t).strip() for t in tags if str(t).strip()]
|
||||
return []
|
||||
return [t.strip() for t in tags.split(",") if t.strip()]
|
||||
|
||||
|
||||
class EntityParser:
|
||||
"""Parser for markdown files into Entity objects."""
|
||||
@@ -68,21 +70,6 @@ class EntityParser:
|
||||
"""Initialize parser with base path for relative permalink generation."""
|
||||
self.base_path = base_path.resolve()
|
||||
|
||||
|
||||
def relative_path(self, file_path: Path) -> str:
|
||||
"""Get file path relative to base_path.
|
||||
|
||||
Example:
|
||||
base_path: /project/root
|
||||
file_path: /project/root/design/models/data.md
|
||||
returns: "design/models/data"
|
||||
"""
|
||||
# Get relative path and remove .md extension
|
||||
rel_path = file_path.resolve().relative_to(self.base_path)
|
||||
if rel_path.suffix.lower() == ".md":
|
||||
return str(rel_path.with_suffix(""))
|
||||
return str(rel_path)
|
||||
|
||||
def parse_date(self, value: Any) -> Optional[datetime]:
|
||||
"""Parse date strings using dateparser for maximum flexibility.
|
||||
|
||||
@@ -96,21 +83,18 @@ class EntityParser:
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
parsed = dateparser.parse(value)
|
||||
if parsed:
|
||||
return parsed
|
||||
except Exception:
|
||||
pass
|
||||
parsed = dateparser.parse(value)
|
||||
if parsed:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
async def parse_file(self, file_path: Path) -> EntityMarkdown:
|
||||
"""Parse markdown file into EntityMarkdown."""
|
||||
|
||||
|
||||
absolute_path = self.base_path / file_path
|
||||
# Parse frontmatter and content using python-frontmatter
|
||||
post = frontmatter.load(str(absolute_path))
|
||||
|
||||
|
||||
# Extract file stat info
|
||||
file_stats = absolute_path.stat()
|
||||
|
||||
@@ -134,4 +118,3 @@ class EntityParser:
|
||||
created=datetime.fromtimestamp(file_stats.st_ctime),
|
||||
modified=datetime.fromtimestamp(file_stats.st_mtime),
|
||||
)
|
||||
|
||||
|
||||
@@ -19,7 +19,7 @@ class DirtyFileError(Exception):
|
||||
|
||||
class MarkdownProcessor:
|
||||
"""Process markdown files while preserving content and structure.
|
||||
|
||||
|
||||
used only for import
|
||||
|
||||
This class handles the file I/O aspects of our markdown processing. It:
|
||||
@@ -90,14 +90,14 @@ class MarkdownProcessor:
|
||||
|
||||
# Convert frontmatter to dict
|
||||
frontmatter_dict = OrderedDict()
|
||||
frontmatter_dict["title"] = markdown.frontmatter.title
|
||||
frontmatter_dict["type"] = markdown.frontmatter.type
|
||||
frontmatter_dict["title"] = markdown.frontmatter.title
|
||||
frontmatter_dict["type"] = markdown.frontmatter.type
|
||||
frontmatter_dict["permalink"] = markdown.frontmatter.permalink
|
||||
|
||||
|
||||
metadata = markdown.frontmatter.metadata or {}
|
||||
for k,v in metadata.items():
|
||||
for k, v in metadata.items():
|
||||
frontmatter_dict[k] = v
|
||||
|
||||
|
||||
# Start with user content (or minimal title for new files)
|
||||
content = markdown.content or f"# {markdown.frontmatter.title}\n"
|
||||
|
||||
@@ -107,7 +107,7 @@ class MarkdownProcessor:
|
||||
# add a blank line if we have semantic content
|
||||
if markdown.observations or markdown.relations:
|
||||
content += "\n"
|
||||
|
||||
|
||||
if markdown.observations:
|
||||
content += self.format_observations(markdown.observations)
|
||||
if markdown.relations:
|
||||
|
||||
@@ -8,19 +8,19 @@ from markdown_it.token import Token
|
||||
# Observation handling functions
|
||||
def is_observation(token: Token) -> bool:
|
||||
"""Check if token looks like our observation format."""
|
||||
if token.type != 'inline':
|
||||
if token.type != "inline": # pragma: no cover
|
||||
return False
|
||||
|
||||
|
||||
content = token.content.strip()
|
||||
if not content:
|
||||
if not content: # pragma: no cover
|
||||
return False
|
||||
|
||||
|
||||
# if it's a markdown_task, return false
|
||||
if content.startswith('[ ]') or content.startswith('[x]') or content.startswith('[-]'):
|
||||
if content.startswith("[ ]") or content.startswith("[x]") or content.startswith("[-]"):
|
||||
return False
|
||||
|
||||
has_category = content.startswith('[') and ']' in content
|
||||
has_tags = '#' in content
|
||||
|
||||
has_category = content.startswith("[") and "]" in content
|
||||
has_tags = "#" in content
|
||||
return has_category or has_tags
|
||||
|
||||
|
||||
@@ -28,119 +28,126 @@ def parse_observation(token: Token) -> Dict[str, Any]:
|
||||
"""Extract observation parts from token."""
|
||||
# Strip bullet point if present
|
||||
content = token.content.strip()
|
||||
if content.startswith('- '):
|
||||
content = content[2:].strip()
|
||||
elif content.startswith('-'):
|
||||
content = content[1:].strip()
|
||||
|
||||
|
||||
# Parse [category]
|
||||
category = None
|
||||
if content.startswith('['):
|
||||
end = content.find(']')
|
||||
if content.startswith("["):
|
||||
end = content.find("]")
|
||||
if end != -1:
|
||||
category = content[1:end].strip() or None # Convert empty to None
|
||||
content = content[end + 1:].strip()
|
||||
|
||||
content = content[end + 1 :].strip()
|
||||
|
||||
# Parse (context)
|
||||
context = None
|
||||
if content.endswith(')'):
|
||||
start = content.rfind('(')
|
||||
if content.endswith(")"):
|
||||
start = content.rfind("(")
|
||||
if start != -1:
|
||||
context = content[start + 1:-1].strip()
|
||||
context = content[start + 1 : -1].strip()
|
||||
content = content[:start].strip()
|
||||
|
||||
# Parse #tags and content
|
||||
|
||||
# Extract tags and keep original content
|
||||
tags = []
|
||||
parts = content.split()
|
||||
content_parts = []
|
||||
tags = set() # Use set to avoid duplicates
|
||||
|
||||
for part in parts:
|
||||
if part.startswith('#'):
|
||||
if part.startswith("#"):
|
||||
# Handle multiple #tags stuck together
|
||||
if '#' in part[1:]:
|
||||
if "#" in part[1:]:
|
||||
# Split on # but keep non-empty tags
|
||||
subtags = [t for t in part.split('#') if t]
|
||||
tags.update(subtags)
|
||||
subtags = [t for t in part.split("#") if t]
|
||||
tags.extend(subtags)
|
||||
else:
|
||||
tags.add(part[1:])
|
||||
else:
|
||||
content_parts.append(part)
|
||||
|
||||
tags.append(part[1:])
|
||||
|
||||
return {
|
||||
'category': category,
|
||||
'content': content,
|
||||
'tags': list(tags) if tags else None,
|
||||
'context': context
|
||||
"category": category,
|
||||
"content": content,
|
||||
"tags": tags if tags else None,
|
||||
"context": context,
|
||||
}
|
||||
|
||||
|
||||
# Relation handling functions
|
||||
def is_explicit_relation(token: Token) -> bool:
|
||||
"""Check if token looks like our relation format."""
|
||||
if token.type != 'inline':
|
||||
if token.type != "inline": # pragma: no cover
|
||||
return False
|
||||
|
||||
|
||||
content = token.content.strip()
|
||||
return '[[' in content and ']]' in content
|
||||
return "[[" in content and "]]" in content
|
||||
|
||||
|
||||
def parse_relation(token: Token) -> Dict[str, Any]:
|
||||
def parse_relation(token: Token) -> Dict[str, Any] | None:
|
||||
"""Extract relation parts from token."""
|
||||
# Remove bullet point if present
|
||||
content = token.content.strip()
|
||||
if content.startswith('- '):
|
||||
content = content[2:].strip()
|
||||
elif content.startswith('-'):
|
||||
content = content[1:].strip()
|
||||
|
||||
|
||||
# Extract [[target]]
|
||||
target = None
|
||||
rel_type = 'relates_to' # default
|
||||
rel_type = "relates_to" # default
|
||||
context = None
|
||||
|
||||
start = content.find('[[')
|
||||
end = content.find(']]')
|
||||
|
||||
|
||||
start = content.find("[[")
|
||||
end = content.find("]]")
|
||||
|
||||
if start != -1 and end != -1:
|
||||
# Get text before link as relation type
|
||||
before = content[:start].strip()
|
||||
if before:
|
||||
rel_type = before
|
||||
|
||||
|
||||
# Get target
|
||||
target = content[start + 2:end].strip()
|
||||
|
||||
target = content[start + 2 : end].strip()
|
||||
|
||||
# Look for context after
|
||||
after = content[end + 2:].strip()
|
||||
if after.startswith('(') and after.endswith(')'):
|
||||
after = content[end + 2 :].strip()
|
||||
if after.startswith("(") and after.endswith(")"):
|
||||
context = after[1:-1].strip() or None
|
||||
|
||||
if not target:
|
||||
|
||||
if not target: # pragma: no cover
|
||||
return None
|
||||
|
||||
return {
|
||||
'type': rel_type,
|
||||
'target': target,
|
||||
'context': context
|
||||
}
|
||||
|
||||
return {"type": rel_type, "target": target, "context": context}
|
||||
|
||||
|
||||
def parse_inline_relations(content: str) -> List[Dict[str, Any]]:
|
||||
"""Find wiki-style links in regular content."""
|
||||
relations = []
|
||||
|
||||
import re
|
||||
pattern = r'\[\[([^\]]+)\]\]'
|
||||
|
||||
for match in re.finditer(pattern, content):
|
||||
target = match.group(1).strip()
|
||||
if target and not target.startswith('[['): # Avoid nested matches
|
||||
relations.append({
|
||||
'type': 'links to',
|
||||
'target': target,
|
||||
'context': None
|
||||
})
|
||||
|
||||
start = 0
|
||||
|
||||
while True:
|
||||
# Find next outer-most [[
|
||||
start = content.find("[[", start)
|
||||
if start == -1: # pragma: no cover
|
||||
break
|
||||
|
||||
# Find matching ]]
|
||||
depth = 1
|
||||
pos = start + 2
|
||||
end = -1
|
||||
|
||||
while pos < len(content):
|
||||
if content[pos : pos + 2] == "[[":
|
||||
depth += 1
|
||||
pos += 2
|
||||
elif content[pos : pos + 2] == "]]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
end = pos
|
||||
break
|
||||
pos += 2
|
||||
else:
|
||||
pos += 1
|
||||
|
||||
if end == -1:
|
||||
# No matching ]] found
|
||||
break
|
||||
|
||||
target = content[start + 2 : end].strip()
|
||||
if target:
|
||||
relations.append({"type": "links to", "target": target, "context": None})
|
||||
|
||||
start = end + 2
|
||||
|
||||
return relations
|
||||
|
||||
|
||||
@@ -149,88 +156,67 @@ def observation_plugin(md: MarkdownIt) -> None:
|
||||
- [category] Content text #tag1 #tag2 (context)
|
||||
- Content text #tag1 (context) # No category is also valid
|
||||
"""
|
||||
|
||||
|
||||
def observation_rule(state: Any) -> None:
|
||||
"""Process observations in token stream."""
|
||||
tokens = state.tokens
|
||||
current_section = None
|
||||
in_list_item = False
|
||||
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
|
||||
# Track current section by headings
|
||||
if token.type == 'heading_open':
|
||||
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
|
||||
if next_token and next_token.type == 'inline':
|
||||
current_section = next_token.content.lower()
|
||||
|
||||
# Track list nesting
|
||||
elif token.type == 'list_item_open':
|
||||
in_list_item = True
|
||||
elif token.type == 'list_item_close':
|
||||
in_list_item = False
|
||||
|
||||
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
|
||||
# Parse observations in list items
|
||||
if token.type == 'inline' and is_observation(token):
|
||||
if token.type == "inline" and is_observation(token):
|
||||
obs = parse_observation(token)
|
||||
if obs['content']: # Only store if we have content
|
||||
token.meta['observation'] = obs
|
||||
|
||||
if obs["content"]: # Only store if we have content
|
||||
token.meta["observation"] = obs
|
||||
|
||||
# Add the rule after inline processing
|
||||
md.core.ruler.after('inline', 'observations', observation_rule)
|
||||
md.core.ruler.after("inline", "observations", observation_rule)
|
||||
|
||||
|
||||
def relation_plugin(md: MarkdownIt) -> None:
|
||||
"""Plugin for parsing relation formats:
|
||||
|
||||
|
||||
Explicit relations:
|
||||
- relation_type [[target]] (context)
|
||||
|
||||
|
||||
Implicit relations (links in content):
|
||||
Some text with [[target]] reference
|
||||
"""
|
||||
|
||||
|
||||
def relation_rule(state: Any) -> None:
|
||||
"""Process relations in token stream."""
|
||||
tokens = state.tokens
|
||||
current_section = None
|
||||
in_list_item = False
|
||||
|
||||
|
||||
for idx in range(len(tokens)):
|
||||
token = tokens[idx]
|
||||
|
||||
# Track current section by headings
|
||||
if token.type == 'heading_open':
|
||||
next_token = tokens[idx + 1] if idx + 1 < len(tokens) else None
|
||||
if next_token and next_token.type == 'inline':
|
||||
current_section = next_token.content.lower()
|
||||
|
||||
|
||||
# Track list nesting
|
||||
elif token.type == 'list_item_open':
|
||||
if token.type == "list_item_open":
|
||||
in_list_item = True
|
||||
elif token.type == 'list_item_close':
|
||||
elif token.type == "list_item_close":
|
||||
in_list_item = False
|
||||
|
||||
|
||||
# Initialize meta for all tokens
|
||||
token.meta = token.meta or {}
|
||||
|
||||
|
||||
# Only process inline tokens
|
||||
if token.type == 'inline':
|
||||
if token.type == "inline":
|
||||
# Check for explicit relations in list items
|
||||
if in_list_item and is_explicit_relation(token):
|
||||
rel = parse_relation(token)
|
||||
if rel:
|
||||
token.meta['relations'] = [rel]
|
||||
|
||||
token.meta["relations"] = [rel]
|
||||
|
||||
# Always check for inline links in any text
|
||||
elif '[[' in token.content:
|
||||
elif "[[" in token.content:
|
||||
rels = parse_inline_relations(token.content)
|
||||
if rels:
|
||||
token.meta['relations'] = token.meta.get('relations', []) + rels
|
||||
|
||||
token.meta["relations"] = token.meta.get("relations", []) + rels
|
||||
|
||||
# Add the rule after inline processing
|
||||
md.core.ruler.after('inline', 'relations', relation_rule)
|
||||
md.core.ruler.after("inline", "relations", relation_rule)
|
||||
|
||||
@@ -13,7 +13,7 @@ class Observation(BaseModel):
|
||||
content: str
|
||||
tags: Optional[List[str]] = None
|
||||
context: Optional[str] = None
|
||||
|
||||
|
||||
def __str__(self) -> str:
|
||||
obs_string = f"- [{self.category}] {self.content}"
|
||||
if self.context:
|
||||
@@ -27,7 +27,7 @@ class Relation(BaseModel):
|
||||
type: str
|
||||
target: str
|
||||
context: Optional[str] = None
|
||||
|
||||
|
||||
def __str__(self) -> str:
|
||||
rel_string = f"- {self.type} [[{self.target}]]"
|
||||
if self.context:
|
||||
@@ -38,24 +38,23 @@ class Relation(BaseModel):
|
||||
class EntityFrontmatter(BaseModel):
|
||||
"""Required frontmatter fields for an entity."""
|
||||
|
||||
metadata: Optional[dict] = None
|
||||
metadata: dict = {}
|
||||
|
||||
@property
|
||||
def tags(self) -> List[str]:
|
||||
return self.metadata.get("tags") if self.metadata else []
|
||||
return self.metadata.get("tags") if self.metadata else [] # pyright: ignore
|
||||
|
||||
@property
|
||||
def title(self) -> str:
|
||||
return self.metadata.get("title") if self.metadata else None
|
||||
return self.metadata.get("title") if self.metadata else None # pyright: ignore
|
||||
|
||||
@property
|
||||
def type(self) -> str:
|
||||
return self.metadata.get("type", "note") if self.metadata else "note"
|
||||
return self.metadata.get("type", "note") if self.metadata else "note" # pyright: ignore
|
||||
|
||||
@property
|
||||
def permalink(self) -> str:
|
||||
return self.metadata.get("permalink") if self.metadata else None
|
||||
|
||||
return self.metadata.get("permalink") if self.metadata else None # pyright: ignore
|
||||
|
||||
|
||||
class EntityMarkdown(BaseModel):
|
||||
|
||||
@@ -1,144 +1,93 @@
|
||||
"""Utilities for converting between markdown and entity models."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
from typing import Optional, Any
|
||||
|
||||
from frontmatter import Post
|
||||
|
||||
from basic_memory.markdown import EntityMarkdown, EntityFrontmatter, Observation, Relation
|
||||
from basic_memory.markdown.entity_parser import parse
|
||||
from basic_memory.models import Entity, ObservationCategory, Observation as ObservationModel
|
||||
from basic_memory.markdown import EntityMarkdown
|
||||
from basic_memory.models import Entity, Observation as ObservationModel
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
|
||||
def entity_model_to_markdown(entity: Entity, content: Optional[str] = None) -> EntityMarkdown:
|
||||
def entity_model_from_markdown(
|
||||
file_path: Path, markdown: EntityMarkdown, entity: Optional[Entity] = None
|
||||
) -> Entity:
|
||||
"""
|
||||
Converts an entity model to its Markdown representation, including metadata,
|
||||
observations, relations, and content. Ensures that observations and relations
|
||||
from the provided content are synchronized with the entity model. Removes
|
||||
duplicate or unmatched observations and relations from the content to maintain
|
||||
consistency.
|
||||
|
||||
:param entity: An instance of the Entity class containing metadata, observations,
|
||||
relations, and other properties of the entity.
|
||||
:type entity: Entity
|
||||
:param content: Optional raw Markdown-formatted content to be parsed for semantic
|
||||
information like observations or relations.
|
||||
:type content: Optional[str]
|
||||
:return: An instance of the EntityMarkdown class containing the entity's
|
||||
frontmatter, observations, relations, and sanitized content formatted
|
||||
in Markdown.
|
||||
:rtype: EntityMarkdown
|
||||
"""
|
||||
metadata = entity.entity_metadata or {}
|
||||
metadata["type"] = entity.entity_type or "note"
|
||||
metadata["title"] = entity.title
|
||||
metadata["permalink"] = entity.permalink
|
||||
|
||||
# convert model to markdown
|
||||
entity_observations = [
|
||||
Observation(
|
||||
category=obs.category,
|
||||
content=obs.content,
|
||||
tags=obs.tags if obs.tags else None,
|
||||
context=obs.context,
|
||||
)
|
||||
for obs in entity.observations
|
||||
]
|
||||
|
||||
entity_relations = [
|
||||
Relation(
|
||||
type=r.relation_type,
|
||||
target=r.to_entity.title if r.to_entity else r.to_name,
|
||||
context=r.context,
|
||||
)
|
||||
for r in entity.outgoing_relations
|
||||
]
|
||||
|
||||
observations = entity_observations
|
||||
relations = entity_relations
|
||||
|
||||
# parse the content to see if it has semantic info (observations/relations)
|
||||
entity_content = parse(content) if content else None
|
||||
|
||||
if entity_content:
|
||||
# remove if they are already in the content
|
||||
observations = [o for o in entity_observations if o not in entity_content.observations]
|
||||
relations = [r for r in entity_relations if r not in entity_content.relations]
|
||||
|
||||
# remove from the content if not present in the db entity
|
||||
for o in entity_content.observations:
|
||||
if o not in entity_observations:
|
||||
content = content.replace(str(o), "")
|
||||
|
||||
for r in entity_content.relations:
|
||||
if r not in entity_relations:
|
||||
content = content.replace(str(r), "")
|
||||
|
||||
return EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(metadata=metadata),
|
||||
content=content,
|
||||
observations=observations,
|
||||
relations=relations,
|
||||
created = entity.created_at,
|
||||
modified = entity.updated_at,
|
||||
)
|
||||
|
||||
|
||||
def entity_model_from_markdown(file_path: Path, markdown: EntityMarkdown, entity: Optional[Entity] = None) -> Entity:
|
||||
"""
|
||||
Convert markdown entity to model.
|
||||
Does not include relations.
|
||||
Convert markdown entity to model. Does not include relations.
|
||||
|
||||
Args:
|
||||
file_path: Path to the markdown file
|
||||
markdown: Parsed markdown entity
|
||||
include_relations: Whether to include relations. Set False for first sync pass.
|
||||
entity: Optional existing entity to update
|
||||
|
||||
Returns:
|
||||
Entity model populated from markdown
|
||||
|
||||
Raises:
|
||||
ValueError: If required datetime fields are missing from markdown
|
||||
"""
|
||||
|
||||
# Validate/default category
|
||||
def get_valid_category(obs):
|
||||
if not obs.category or obs.category not in [c.value for c in ObservationCategory]:
|
||||
return ObservationCategory.NOTE.value
|
||||
return obs.category
|
||||
if not markdown.created or not markdown.modified: # pragma: no cover
|
||||
raise ValueError("Both created and modified dates are required in markdown")
|
||||
|
||||
# Generate permalink if not provided
|
||||
permalink = markdown.frontmatter.permalink or generate_permalink(file_path)
|
||||
|
||||
# Create or update entity
|
||||
model = entity or Entity()
|
||||
|
||||
model.title=markdown.frontmatter.title
|
||||
model.entity_type=markdown.frontmatter.type
|
||||
model.permalink=permalink
|
||||
model.file_path=str(file_path)
|
||||
model.content_type="text/markdown"
|
||||
model.created_at=markdown.created
|
||||
model.updated_at=markdown.modified
|
||||
model.entity_metadata={k:str(v) for k,v in markdown.frontmatter.metadata.items()}
|
||||
model.observations=[
|
||||
ObservationModel(
|
||||
content=obs.content,
|
||||
category=get_valid_category(obs),
|
||||
context=obs.context,
|
||||
tags=obs.tags,
|
||||
)
|
||||
for obs in markdown.observations
|
||||
]
|
||||
|
||||
|
||||
# Update basic fields
|
||||
model.title = markdown.frontmatter.title
|
||||
model.entity_type = markdown.frontmatter.type
|
||||
model.permalink = permalink
|
||||
model.file_path = str(file_path)
|
||||
model.content_type = "text/markdown"
|
||||
model.created_at = markdown.created
|
||||
model.updated_at = markdown.modified
|
||||
|
||||
# Handle metadata - ensure all values are strings and filter None
|
||||
metadata = markdown.frontmatter.metadata or {}
|
||||
model.entity_metadata = {k: str(v) for k, v in metadata.items() if v is not None}
|
||||
|
||||
# Convert observations
|
||||
model.observations = [
|
||||
ObservationModel(
|
||||
content=obs.content,
|
||||
category=obs.category,
|
||||
context=obs.context,
|
||||
tags=obs.tags,
|
||||
)
|
||||
for obs in markdown.observations
|
||||
]
|
||||
|
||||
return model
|
||||
|
||||
async def schema_to_markdown(schema):
|
||||
|
||||
async def schema_to_markdown(schema: Any) -> Post:
|
||||
"""
|
||||
Convert schema to markdown.
|
||||
:param schema: the schema to convert
|
||||
:return: Post
|
||||
Convert schema to markdown Post object.
|
||||
|
||||
Args:
|
||||
schema: Schema to convert (must have title, entity_type, and permalink attributes)
|
||||
|
||||
Returns:
|
||||
Post object with frontmatter metadata
|
||||
"""
|
||||
# Create Post object
|
||||
# Extract content and metadata
|
||||
content = schema.content or ""
|
||||
frontmatter_metadata = schema.entity_metadata or {}
|
||||
|
||||
# remove from map so we can define ordering in frontmatter
|
||||
if "type" in frontmatter_metadata:
|
||||
del frontmatter_metadata["type"]
|
||||
if "title" in frontmatter_metadata:
|
||||
del frontmatter_metadata["title"]
|
||||
if "permalink" in frontmatter_metadata:
|
||||
del frontmatter_metadata["permalink"]
|
||||
|
||||
post = Post(content, title=schema.title, type=schema.entity_type, permalink=schema.permalink, **frontmatter_metadata)
|
||||
frontmatter_metadata = dict(schema.entity_metadata or {})
|
||||
|
||||
# Remove special fields for ordered frontmatter
|
||||
for field in ["type", "title", "permalink"]:
|
||||
frontmatter_metadata.pop(field, None)
|
||||
|
||||
# Create Post with ordered fields
|
||||
post = Post(
|
||||
content,
|
||||
title=schema.title,
|
||||
type=schema.entity_type,
|
||||
permalink=schema.permalink,
|
||||
**frontmatter_metadata,
|
||||
)
|
||||
return post
|
||||
|
||||
@@ -1 +1 @@
|
||||
"""MCP server for basic-memory."""
|
||||
"""MCP server for basic-memory."""
|
||||
|
||||
@@ -2,9 +2,7 @@ from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from basic_memory.api.app import app as fastapi_app
|
||||
|
||||
BASE_URL = "http://test"
|
||||
BASE_URL = "memory://"
|
||||
|
||||
# Create shared async client
|
||||
client = AsyncClient(transport=ASGITransport(app=fastapi_app), base_url=BASE_URL)
|
||||
|
||||
|
||||
|
||||
@@ -1,21 +0,0 @@
|
||||
"""Main MCP entrypoint for Basic Memory.
|
||||
|
||||
Creates and configures the shared MCP instance and handles server startup.
|
||||
"""
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.config import config
|
||||
|
||||
# Import shared mcp instance
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
# Import tools to register them
|
||||
import basic_memory.mcp.tools # noqa: F401
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
home_dir = config.home
|
||||
logger.info("Starting Basic Memory MCP server")
|
||||
logger.info(f"Home directory: {home_dir}")
|
||||
mcp.run()
|
||||
@@ -1,39 +1,15 @@
|
||||
"""Enhanced FastMCP server instance for Basic Memory."""
|
||||
import sys
|
||||
|
||||
from loguru import logger
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.server.fastmcp.utilities.logging import configure_logging
|
||||
|
||||
from basic_memory.config import config
|
||||
from basic_memory.utils import setup_logging
|
||||
|
||||
# mcp console logging
|
||||
configure_logging(level="INFO")
|
||||
# configure_logging(level='INFO')
|
||||
|
||||
|
||||
def setup_logging(home_dir: str = config.home, log_file: str = ".basic-memory/basic-memory.log"):
|
||||
"""Configure file logging to the basic-memory home directory."""
|
||||
log = f"{home_dir}/{log_file}"
|
||||
|
||||
# Add file handler with rotation
|
||||
logger.add(
|
||||
log,
|
||||
rotation="100 MB",
|
||||
retention="10 days",
|
||||
backtrace=True,
|
||||
diagnose=True,
|
||||
enqueue=True,
|
||||
colorize=False,
|
||||
)
|
||||
|
||||
# Add stderr handler
|
||||
logger.add(
|
||||
sys.stderr,
|
||||
colorize=True,
|
||||
)
|
||||
|
||||
# start our out file logging
|
||||
setup_logging()
|
||||
setup_logging(log_file=".basic-memory/basic-memory.log")
|
||||
|
||||
# Create the shared server instance
|
||||
mcp = FastMCP("Basic Memory")
|
||||
|
||||
@@ -7,8 +7,10 @@ all tools with the MCP server.
|
||||
|
||||
# Import tools to register them with MCP
|
||||
from basic_memory.mcp.tools.memory import build_context, recent_activity
|
||||
#from basic_memory.mcp.tools.ai_edit import ai_edit
|
||||
|
||||
# from basic_memory.mcp.tools.ai_edit import ai_edit
|
||||
from basic_memory.mcp.tools.notes import read_note, write_note
|
||||
from basic_memory.mcp.tools.search import search
|
||||
|
||||
from basic_memory.mcp.tools.knowledge import (
|
||||
delete_entities,
|
||||
@@ -26,9 +28,9 @@ __all__ = [
|
||||
# memory tools
|
||||
"build_context",
|
||||
"recent_activity",
|
||||
#notes
|
||||
# notes
|
||||
"read_note",
|
||||
"write_note",
|
||||
# file edit
|
||||
#"ai_edit",
|
||||
# "ai_edit",
|
||||
]
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
"""Tool for AI-assisted file editing."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
|
||||
|
||||
def _detect_indent(text: str, match_pos: int) -> int:
|
||||
"""Get indentation level at a position in text."""
|
||||
# Find start of line containing the match
|
||||
line_start = text.rfind("\n", 0, match_pos)
|
||||
if line_start < 0:
|
||||
line_start = 0
|
||||
else:
|
||||
line_start += 1 # Skip newline char
|
||||
|
||||
# Count leading spaces
|
||||
pos = line_start
|
||||
while pos < len(text) and text[pos].isspace():
|
||||
pos += 1
|
||||
return pos - line_start
|
||||
|
||||
|
||||
def _apply_indent(text: str, spaces: int) -> str:
|
||||
"""Apply indentation to text."""
|
||||
prefix = " " * spaces
|
||||
return "\n".join(prefix + line if line.strip() else line for line in text.split("\n"))
|
||||
|
||||
|
||||
@mcp.tool()
|
||||
async def ai_edit(path: str, edits: List[Dict[str, Any]]) -> bool:
|
||||
"""AI-assisted file editing tool.
|
||||
|
||||
Args:
|
||||
path: Path to file to edit
|
||||
edits: List of edits to apply. Each edit is a dict with:
|
||||
oldText: Text to replace
|
||||
newText: New content
|
||||
options: Optional dict with:
|
||||
indent: Number of spaces to indent
|
||||
preserveIndentation: Keep existing indent (default: true)
|
||||
|
||||
Returns:
|
||||
bool: True if edits were applied successfully
|
||||
"""
|
||||
try:
|
||||
# Read file
|
||||
content = Path(path).read_text()
|
||||
original = content
|
||||
success = True
|
||||
|
||||
# Apply each edit
|
||||
for edit in edits:
|
||||
old_text = edit["oldText"]
|
||||
new_text = edit["newText"]
|
||||
options = edit.get("options", {})
|
||||
|
||||
# Find text to replace
|
||||
match_pos = content.find(old_text)
|
||||
if match_pos < 0:
|
||||
success = False
|
||||
continue
|
||||
|
||||
# Handle indentation
|
||||
if not options.get("preserveIndentation", True):
|
||||
# Use existing indentation
|
||||
indent = _detect_indent(content, match_pos)
|
||||
new_text = _apply_indent(new_text, indent)
|
||||
elif "indent" in options:
|
||||
# Use specified indentation
|
||||
new_text = _apply_indent(new_text, options["indent"])
|
||||
|
||||
# Apply the edit
|
||||
content = content.replace(old_text, new_text)
|
||||
|
||||
# Write back if changed
|
||||
if content != original:
|
||||
Path(path).write_text(content)
|
||||
return success
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error applying edits: {e}")
|
||||
return False
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_get, call_post
|
||||
from basic_memory.schemas.base import PathId
|
||||
from basic_memory.schemas.base import Permalink
|
||||
from basic_memory.schemas.request import (
|
||||
GetEntitiesRequest,
|
||||
)
|
||||
@@ -16,7 +16,7 @@ from basic_memory.mcp.async_client import client
|
||||
@mcp.tool(
|
||||
description="Get complete information about a specific entity including observations and relations",
|
||||
)
|
||||
async def get_entity(permalink: PathId) -> EntityResponse:
|
||||
async def get_entity(permalink: Permalink) -> EntityResponse:
|
||||
"""Get a specific entity info by its permalink.
|
||||
|
||||
Args:
|
||||
|
||||
@@ -7,9 +7,13 @@ from loguru import logger
|
||||
from basic_memory.mcp.async_client import client
|
||||
from basic_memory.mcp.server import mcp
|
||||
from basic_memory.mcp.tools.utils import call_get
|
||||
from basic_memory.schemas.memory import GraphContext, MemoryUrl, memory_url, memory_url_path, normalize_memory_url
|
||||
from basic_memory.schemas.memory import (
|
||||
GraphContext,
|
||||
MemoryUrl,
|
||||
memory_url_path,
|
||||
normalize_memory_url,
|
||||
)
|
||||
from basic_memory.schemas.base import TimeFrame
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
@@ -84,7 +88,7 @@ async def build_context(
|
||||
""",
|
||||
)
|
||||
async def recent_activity(
|
||||
type: List[Literal["entity", "observation", "relation"]] = None,
|
||||
type: List[Literal["entity", "observation", "relation"]] = [],
|
||||
depth: Optional[int] = 1,
|
||||
timeframe: Optional[TimeFrame] = "7d",
|
||||
max_results: int = 10,
|
||||
@@ -136,7 +140,7 @@ async def recent_activity(
|
||||
"timeframe": timeframe,
|
||||
"max_results": max_results,
|
||||
}
|
||||
if type:
|
||||
if type:
|
||||
params["type"] = type
|
||||
|
||||
response = await call_get(
|
||||
|
||||
@@ -13,6 +13,7 @@ from basic_memory.mcp.async_client import client
|
||||
from basic_memory.schemas import EntityResponse, DeleteEntitiesResponse
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.mcp.tools.utils import call_get, call_put, call_delete
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
@@ -96,7 +97,9 @@ async def read_note(identifier: str) -> str:
|
||||
Raises:
|
||||
ValueError: If the note cannot be found
|
||||
"""
|
||||
response = await call_get(client, f"/resource/{identifier}")
|
||||
logger.info(f"Reading note {identifier}")
|
||||
url = memory_url_path(identifier)
|
||||
response = await call_get(client, f"/resource/{url}")
|
||||
return response.text
|
||||
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Search tools for Basic Memory MCP server."""
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.mcp.server import mcp
|
||||
@@ -24,5 +25,5 @@ async def search(query: SearchQuery) -> SearchResponse:
|
||||
SearchResponse with search results and metadata
|
||||
"""
|
||||
logger.info(f"Searching for {query.text}")
|
||||
response = await call_post(client,"/search/", json=query.model_dump())
|
||||
response = await call_post(client, "/search/", json=query.model_dump())
|
||||
return SearchResponse.model_validate(response.json())
|
||||
|
||||
@@ -45,7 +45,7 @@ async def call_get(
|
||||
return response
|
||||
except HTTPStatusError as e:
|
||||
logger.error(f"Error calling GET {url}: {e}")
|
||||
raise ToolError(f"Error calling tool: {e}. Response: {response.text}") from e
|
||||
raise ToolError(f"Error calling tool: {e}.") from e
|
||||
|
||||
|
||||
async def call_put(
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import basic_memory
|
||||
from basic_memory.models.base import Base
|
||||
from basic_memory.models.knowledge import Entity, Observation, Relation, ObservationCategory
|
||||
from basic_memory.models.knowledge import Entity, Observation, Relation
|
||||
|
||||
SCHEMA_VERSION = basic_memory.__version__ + "-" + "003"
|
||||
|
||||
@@ -10,6 +10,5 @@ __all__ = [
|
||||
"Base",
|
||||
"Entity",
|
||||
"Observation",
|
||||
"ObservationCategory",
|
||||
"Relation",
|
||||
]
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""Base model class for SQLAlchemy models."""
|
||||
from sqlalchemy import String, Integer
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncAttrs
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
|
||||
class Base(AsyncAttrs, DeclarativeBase):
|
||||
"""Base class for all models"""
|
||||
pass
|
||||
|
||||
pass
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Knowledge graph models."""
|
||||
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
@@ -14,17 +13,15 @@ from sqlalchemy import (
|
||||
Index,
|
||||
JSON,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship, validates
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from basic_memory.models.base import Base
|
||||
from enum import Enum
|
||||
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
|
||||
class Entity(Base):
|
||||
"""
|
||||
Core entity in the knowledge graph.
|
||||
"""Core entity in the knowledge graph.
|
||||
|
||||
Entities represent semantic nodes maintained by the AI layer. Each entity:
|
||||
- Has a unique numeric ID (database-generated)
|
||||
@@ -79,44 +76,15 @@ class Entity(Base):
|
||||
|
||||
@property
|
||||
def relations(self):
|
||||
"""Get all relations (incoming and outgoing) for this entity."""
|
||||
return self.incoming_relations + self.outgoing_relations
|
||||
|
||||
@validates("permalink")
|
||||
def validate_permalink(self, key, value):
|
||||
"""Validate permalink format.
|
||||
|
||||
Requirements:
|
||||
1. Must be valid URI path component
|
||||
2. Only lowercase letters, numbers, and hyphens (no underscores)
|
||||
3. Path segments separated by forward slashes
|
||||
4. No leading/trailing hyphens in segments
|
||||
"""
|
||||
if not value:
|
||||
raise ValueError("Permalink must not be None")
|
||||
|
||||
if not re.match(r"^[a-z0-9][a-z0-9\-/]*[a-z0-9]$", value):
|
||||
raise ValueError(
|
||||
f"Invalid permalink format: {value}. "
|
||||
"Use only lowercase letters, numbers, and hyphens."
|
||||
)
|
||||
return value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Entity(id={self.id}, name='{self.title}', type='{self.entity_type}'"
|
||||
|
||||
|
||||
class ObservationCategory(str, Enum):
|
||||
TECH = "tech"
|
||||
DESIGN = "design"
|
||||
FEATURE = "feature"
|
||||
NOTE = "note"
|
||||
ISSUE = "issue"
|
||||
TODO = "todo"
|
||||
|
||||
|
||||
class Observation(Base):
|
||||
"""
|
||||
An observation about an entity.
|
||||
"""An observation about an entity.
|
||||
|
||||
Observations are atomic facts or notes about an entity.
|
||||
"""
|
||||
@@ -130,13 +98,8 @@ class Observation(Base):
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
entity_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
|
||||
content: Mapped[str] = mapped_column(Text)
|
||||
category: Mapped[str] = mapped_column(
|
||||
String,
|
||||
nullable=False,
|
||||
default=ObservationCategory.NOTE.value,
|
||||
server_default=ObservationCategory.NOTE.value,
|
||||
)
|
||||
context: Mapped[str] = mapped_column(Text, nullable=True)
|
||||
category: Mapped[str] = mapped_column(String, nullable=False, default="note")
|
||||
context: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
tags: Mapped[Optional[list[str]]] = mapped_column(
|
||||
JSON, nullable=True, default=list, server_default="[]"
|
||||
)
|
||||
@@ -146,23 +109,21 @@ class Observation(Base):
|
||||
|
||||
@property
|
||||
def permalink(self) -> str:
|
||||
"""
|
||||
Create synthetic permalink for the observation
|
||||
We can construct these because observations are always
|
||||
defined in and owned by a single entity
|
||||
"""Create synthetic permalink for the observation.
|
||||
|
||||
We can construct these because observations are always defined in
|
||||
and owned by a single entity.
|
||||
"""
|
||||
return generate_permalink(
|
||||
f"{self.entity.permalink}/observations/{self.category}/{self.content}"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
def __repr__(self) -> str: # pragma: no cover
|
||||
return f"Observation(id={self.id}, entity_id={self.entity_id}, content='{self.content}')"
|
||||
|
||||
|
||||
class Relation(Base):
|
||||
"""
|
||||
A directed relation between two entities.
|
||||
"""
|
||||
"""A directed relation between two entities."""
|
||||
|
||||
__tablename__ = "relation"
|
||||
__table_args__ = (
|
||||
@@ -174,12 +135,12 @@ class Relation(Base):
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
from_id: Mapped[int] = mapped_column(Integer, ForeignKey("entity.id", ondelete="CASCADE"))
|
||||
to_id: Mapped[int] = mapped_column(
|
||||
to_id: Mapped[Optional[int]] = mapped_column(
|
||||
Integer, ForeignKey("entity.id", ondelete="CASCADE"), nullable=True
|
||||
)
|
||||
to_name: Mapped[str] = mapped_column(String)
|
||||
relation_type: Mapped[str] = mapped_column(String)
|
||||
context: Mapped[str] = mapped_column(Text, nullable=True)
|
||||
context: Mapped[Optional[str]] = mapped_column(Text, nullable=True)
|
||||
|
||||
# Relationships
|
||||
from_entity = relationship(
|
||||
@@ -189,15 +150,17 @@ class Relation(Base):
|
||||
|
||||
@property
|
||||
def permalink(self) -> str:
|
||||
"""Create relation permalink showing the semantic connection:
|
||||
source/relation_type/target
|
||||
e.g., "specs/search/implements/features/search-ui"
|
||||
"""
|
||||
"""Create relation permalink showing the semantic connection.
|
||||
|
||||
Format: source/relation_type/target
|
||||
Example: "specs/search/implements/features/search-ui"
|
||||
"""
|
||||
if self.to_entity:
|
||||
return generate_permalink(
|
||||
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
|
||||
)
|
||||
return generate_permalink(
|
||||
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_entity.permalink}"
|
||||
if self.to_entity
|
||||
else f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
|
||||
f"{self.from_entity.permalink}/{self.relation_type}/{self.to_name}"
|
||||
)
|
||||
|
||||
def __repr__(self) -> str:
|
||||
|
||||
@@ -31,4 +31,4 @@ CREATE VIRTUAL TABLE IF NOT EXISTS search_index USING fts5(
|
||||
tokenize='unicode61 tokenchars 0x2F', -- Hex code for /
|
||||
prefix='1,2,3,4' -- Support longer prefixes for paths
|
||||
);
|
||||
""")
|
||||
""")
|
||||
|
||||
@@ -2,6 +2,8 @@ from .entity_repository import EntityRepository
|
||||
from .observation_repository import ObservationRepository
|
||||
from .relation_repository import RelationRepository
|
||||
|
||||
__all__ = ["EntityRepository", "ObservationRepository", "RelationRepository", ]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"EntityRepository",
|
||||
"ObservationRepository",
|
||||
"RelationRepository",
|
||||
]
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
"""Repository for managing entities in the knowledge graph."""
|
||||
|
||||
from typing import List, Optional, Sequence
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Sequence, Union
|
||||
|
||||
from sqlalchemy import select, or_, asc
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
from sqlalchemy.orm import selectinload
|
||||
from sqlalchemy.orm.interfaces import LoaderOption
|
||||
@@ -12,109 +12,57 @@ from basic_memory.repository.repository import Repository
|
||||
|
||||
|
||||
class EntityRepository(Repository[Entity]):
|
||||
"""Repository for Entity model."""
|
||||
"""Repository for Entity model.
|
||||
|
||||
Note: All file paths are stored as strings in the database. Convert Path objects
|
||||
to strings before passing to repository methods.
|
||||
"""
|
||||
|
||||
def __init__(self, session_maker: async_sessionmaker[AsyncSession]):
|
||||
"""Initialize with session maker."""
|
||||
super().__init__(session_maker, Entity)
|
||||
|
||||
async def get_by_permalink(self, permalink: str) -> Optional[Entity]:
|
||||
"""Get entity by permalink."""
|
||||
"""Get entity by permalink.
|
||||
|
||||
Args:
|
||||
permalink: Unique identifier for the entity
|
||||
"""
|
||||
query = self.select().where(Entity.permalink == permalink).options(*self.get_load_options())
|
||||
return await self.find_one(query)
|
||||
|
||||
async def get_by_title(self, title: str) -> Optional[Entity]:
|
||||
"""Get entity by title."""
|
||||
"""Get entity by title.
|
||||
|
||||
Args:
|
||||
title: Title of the entity to find
|
||||
"""
|
||||
query = self.select().where(Entity.title == title).options(*self.get_load_options())
|
||||
return await self.find_one(query)
|
||||
|
||||
async def get_by_file_path(self, file_path: str) -> Optional[Entity]:
|
||||
"""Get entity by file_path."""
|
||||
query = self.select().where(Entity.file_path == file_path).options(*self.get_load_options())
|
||||
return await self.find_one(query)
|
||||
async def get_by_file_path(self, file_path: Union[Path, str]) -> Optional[Entity]:
|
||||
"""Get entity by file_path.
|
||||
|
||||
async def list_entities(
|
||||
self,
|
||||
entity_type: Optional[str] = None,
|
||||
sort_by: Optional[str] = "updated_at",
|
||||
include_related: bool = False,
|
||||
) -> Sequence[Entity]:
|
||||
"""List all entities, optionally filtered by type and sorted."""
|
||||
query = self.select()
|
||||
|
||||
# Always load base relations
|
||||
query = query.options(*self.get_load_options())
|
||||
|
||||
# Apply filters
|
||||
if entity_type:
|
||||
# When include_related is True, get both:
|
||||
# 1. Entities of the requested type
|
||||
# 2. Entities that have relations with entities of the requested type
|
||||
if include_related:
|
||||
query = query.where(
|
||||
or_(
|
||||
Entity.entity_type == entity_type,
|
||||
Entity.outgoing_relations.any(
|
||||
Relation.to_entity.has(entity_type=entity_type)
|
||||
),
|
||||
Entity.incoming_relations.any(
|
||||
Relation.from_entity.has(entity_type=entity_type)
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
query = query.where(Entity.entity_type == entity_type)
|
||||
|
||||
# Apply sorting
|
||||
if sort_by:
|
||||
sort_field = getattr(Entity, sort_by, Entity.updated_at)
|
||||
query = query.order_by(asc(sort_field))
|
||||
|
||||
result = await self.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def get_entity_types(self) -> List[str]:
|
||||
"""Get list of distinct entity types."""
|
||||
query = select(Entity.entity_type).distinct()
|
||||
|
||||
result = await self.execute_query(query, use_query_options=False)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def search(self, query_str: str) -> List[Entity]:
|
||||
Args:
|
||||
file_path: Path to the entity file (will be converted to string internally)
|
||||
"""
|
||||
Search for entities.
|
||||
|
||||
Searches across:
|
||||
- Entity names
|
||||
- Entity types
|
||||
- Entity descriptions
|
||||
- Associated Observations content
|
||||
"""
|
||||
search_term = f"%{query_str}%"
|
||||
query = (
|
||||
self.select()
|
||||
.where(
|
||||
or_(
|
||||
Entity.title.ilike(search_term),
|
||||
Entity.entity_type.ilike(search_term),
|
||||
Entity.summary.ilike(search_term),
|
||||
Entity.observations.any(Observation.content.ilike(search_term)),
|
||||
)
|
||||
)
|
||||
.where(Entity.file_path == str(file_path))
|
||||
.options(*self.get_load_options())
|
||||
)
|
||||
result = await self.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
return await self.find_one(query)
|
||||
|
||||
async def delete_entities_by_doc_id(self, doc_id: int) -> bool:
|
||||
"""Delete all entities associated with a document."""
|
||||
return await self.delete_by_fields(doc_id=doc_id)
|
||||
async def delete_by_file_path(self, file_path: Union[Path, str]) -> bool:
|
||||
"""Delete entity with the provided file_path.
|
||||
|
||||
async def delete_by_file_path(self, file_path: str) -> bool:
|
||||
"""Delete entity with the provided file_path."""
|
||||
return await self.delete_by_fields(file_path=file_path)
|
||||
Args:
|
||||
file_path: Path to the entity file (will be converted to string internally)
|
||||
"""
|
||||
return await self.delete_by_fields(file_path=str(file_path))
|
||||
|
||||
def get_load_options(self) -> List[LoaderOption]:
|
||||
"""Get SQLAlchemy loader options for eager loading relationships."""
|
||||
return [
|
||||
selectinload(Entity.observations).selectinload(Observation.entity),
|
||||
# Load from_relations and both entities for each relation
|
||||
@@ -126,8 +74,11 @@ class EntityRepository(Repository[Entity]):
|
||||
]
|
||||
|
||||
async def find_by_permalinks(self, permalinks: List[str]) -> Sequence[Entity]:
|
||||
"""Find multiple entities by their permalink."""
|
||||
"""Find multiple entities by their permalink.
|
||||
|
||||
Args:
|
||||
permalinks: List of permalink strings to find
|
||||
"""
|
||||
# Handle empty input explicitly
|
||||
if not permalinks:
|
||||
return []
|
||||
@@ -139,18 +90,3 @@ class EntityRepository(Repository[Entity]):
|
||||
|
||||
result = await self.execute_query(query)
|
||||
return list(result.scalars().all())
|
||||
|
||||
async def delete_by_permalinks(self, permalinks: List[str]) -> int:
|
||||
"""Delete multiple entities by permalink."""
|
||||
|
||||
# Handle empty input explicitly
|
||||
if not permalinks:
|
||||
return 0
|
||||
|
||||
# Find matching entities
|
||||
entities = await self.find_by_permalinks(permalinks)
|
||||
if not entities:
|
||||
return 0
|
||||
|
||||
# Use existing delete_by_ids
|
||||
return await self.delete_by_ids([entity.id for entity in entities])
|
||||
|
||||
@@ -40,12 +40,6 @@ class RelationRepository(Repository[Relation]):
|
||||
)
|
||||
return await self.find_one(query)
|
||||
|
||||
async def find_by_entity(self, from_entity_id: int) -> Sequence[Relation]:
|
||||
"""Find all relations from a specific entity."""
|
||||
query = select(Relation).filter(Relation.from_id == from_entity_id)
|
||||
result = await self.execute_query(query)
|
||||
return result.scalars().all()
|
||||
|
||||
async def find_by_entities(self, from_id: int, to_id: int) -> Sequence[Relation]:
|
||||
"""Find all relations between two entities."""
|
||||
query = select(Relation).where((Relation.from_id == from_id) & (Relation.to_id == to_id))
|
||||
@@ -73,6 +67,5 @@ class RelationRepository(Repository[Relation]):
|
||||
result = await self.execute_query(query)
|
||||
return result.scalars().all()
|
||||
|
||||
|
||||
def get_load_options(self) -> List[LoaderOption]:
|
||||
return [selectinload(Relation.from_entity), selectinload(Relation.to_entity)]
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
"""Base repository implementation."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Type, Optional, Any, Sequence, TypeVar, List
|
||||
|
||||
from loguru import logger
|
||||
@@ -98,13 +97,6 @@ class Repository[T: Base]:
|
||||
entities = (self.Model,)
|
||||
return select(*entities)
|
||||
|
||||
async def refresh(self, instance: T, relationships: list[str] | None = None) -> None:
|
||||
"""Refresh instance and optionally specified relationships."""
|
||||
logger.debug(f"Refreshing {self.Model.__name__} instance: {getattr(instance, 'id', None)}")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await session.refresh(instance, relationships or [])
|
||||
logger.debug(f"Refreshed relationships: {relationships}")
|
||||
|
||||
async def find_all(self, skip: int = 0, limit: Optional[int] = 0) -> Sequence[T]:
|
||||
"""Fetch records from the database with pagination."""
|
||||
logger.debug(f"Finding all {self.Model.__name__} (skip={skip}, limit={limit})")
|
||||
@@ -149,35 +141,6 @@ class Repository[T: Base]:
|
||||
logger.debug(f"No {self.Model.__name__} found")
|
||||
return entity
|
||||
|
||||
async def find_modified_since(self, since: datetime) -> Sequence[T]:
|
||||
"""Find all records modified since the given timestamp.
|
||||
|
||||
This method assumes the model has an updated_at column. Override
|
||||
in subclasses if a different column should be used.
|
||||
|
||||
Args:
|
||||
since: Datetime to search from
|
||||
|
||||
Returns:
|
||||
Sequence of records modified since the timestamp
|
||||
"""
|
||||
logger.debug(f"Finding {self.Model.__name__} modified since: {since}")
|
||||
|
||||
if not hasattr(self.Model, "updated_at"):
|
||||
raise AttributeError(f"{self.Model.__name__} does not have updated_at column")
|
||||
|
||||
query = (
|
||||
select(self.Model)
|
||||
.filter(self.Model.updated_at >= since)
|
||||
.options(*self.get_load_options())
|
||||
)
|
||||
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
result = await session.execute(query)
|
||||
items = result.scalars().all()
|
||||
logger.debug(f"Found {len(items)} modified {self.Model.__name__} records")
|
||||
return items
|
||||
|
||||
async def create(self, data: dict) -> T:
|
||||
"""Create a new record from a model instance."""
|
||||
logger.debug(f"Creating {self.Model.__name__} from entity_data: {data}")
|
||||
@@ -223,11 +186,11 @@ class Repository[T: Base]:
|
||||
for key, value in entity_data.items():
|
||||
if key in self.valid_columns:
|
||||
setattr(entity, key, value)
|
||||
|
||||
|
||||
elif isinstance(entity_data, self.Model):
|
||||
for column in self.Model.__table__.columns.keys():
|
||||
setattr(entity, column, getattr(entity_data, column))
|
||||
|
||||
|
||||
await session.flush() # Make sure changes are flushed
|
||||
await session.refresh(entity) # Refresh
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ class SearchIndexRow:
|
||||
|
||||
id: int
|
||||
type: str
|
||||
permalink: str
|
||||
file_path: str
|
||||
metadata: Optional[dict] = None
|
||||
|
||||
# date values
|
||||
@@ -30,13 +32,9 @@ class SearchIndexRow:
|
||||
# assigned in result
|
||||
score: Optional[float] = None
|
||||
|
||||
# Common fields
|
||||
permalink: Optional[str] = None
|
||||
file_path: Optional[str] = None
|
||||
|
||||
# Type-specific fields
|
||||
title: Optional[int] = None # entity
|
||||
content: Optional[int] = None # entity, observation
|
||||
title: Optional[str] = None # entity
|
||||
content: Optional[str] = None # entity, observation
|
||||
entity_id: Optional[int] = None # observations
|
||||
category: Optional[str] = None # observations
|
||||
from_id: Optional[int] = None # relations
|
||||
@@ -70,6 +68,8 @@ class SearchRepository:
|
||||
|
||||
async def init_search_index(self):
|
||||
"""Create or recreate the search index."""
|
||||
|
||||
logger.info("Initializing search index")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await session.execute(CREATE_SEARCH_INDEX)
|
||||
await session.commit()
|
||||
@@ -79,8 +79,8 @@ class SearchRepository:
|
||||
For FTS5, special characters and phrases need to be quoted to be treated as a single token.
|
||||
"""
|
||||
# List of special characters that need quoting
|
||||
special_chars = ['/', '*', '-', '.', ' ', '(', ')', '[', ']', '"', "'"]
|
||||
|
||||
special_chars = ["/", "*", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
|
||||
|
||||
# Check if term contains any special characters
|
||||
if any(c in term for c in special_chars):
|
||||
# If the term already contains quotes, escape them
|
||||
@@ -94,16 +94,16 @@ class SearchRepository:
|
||||
permalink: Optional[str] = None,
|
||||
permalink_match: Optional[str] = None,
|
||||
title: Optional[str] = None,
|
||||
types: List[SearchItemType] = None,
|
||||
types: Optional[List[SearchItemType]] = None,
|
||||
after_date: Optional[datetime] = None,
|
||||
entity_types: List[str] = None,
|
||||
entity_types: Optional[List[str]] = None,
|
||||
limit: int = 10,
|
||||
) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content with fuzzy matching."""
|
||||
conditions = []
|
||||
params = {}
|
||||
order_by_clause = ""
|
||||
|
||||
|
||||
# Handle text search for title and content
|
||||
if search_text:
|
||||
search_text = self._quote_search_term(search_text.lower().strip())
|
||||
@@ -125,7 +125,7 @@ class SearchRepository:
|
||||
if permalink_match:
|
||||
params["permalink"] = self._quote_search_term(permalink_match)
|
||||
conditions.append("permalink MATCH :permalink")
|
||||
|
||||
|
||||
# Handle type filter
|
||||
if types:
|
||||
type_list = ", ".join(f"'{t.value}'" for t in types)
|
||||
@@ -140,13 +140,13 @@ class SearchRepository:
|
||||
if after_date:
|
||||
params["after_date"] = after_date
|
||||
conditions.append("datetime(created_at) > datetime(:after_date)")
|
||||
|
||||
|
||||
# order by most recent first
|
||||
order_by_clause = ", updated_at DESC"
|
||||
|
||||
# set limit on search query
|
||||
params["limit"] = limit
|
||||
|
||||
|
||||
# Build WHERE clause
|
||||
where_clause = " AND ".join(conditions) if conditions else "1=1"
|
||||
|
||||
@@ -173,7 +173,7 @@ class SearchRepository:
|
||||
LIMIT :limit
|
||||
"""
|
||||
|
||||
#logger.debug(f"Search {sql} params: {params}")
|
||||
# logger.debug(f"Search {sql} params: {params}")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
result = await session.execute(text(sql), params)
|
||||
rows = result.fetchall()
|
||||
@@ -199,9 +199,9 @@ class SearchRepository:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
#for r in results:
|
||||
# for r in results:
|
||||
# logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
|
||||
|
||||
|
||||
return results
|
||||
|
||||
async def index_item(
|
||||
@@ -248,17 +248,14 @@ class SearchRepository:
|
||||
async def execute_query(
|
||||
self,
|
||||
query: Executable,
|
||||
params: Optional[Dict[str, Any]] = None,
|
||||
params: Dict[str, Any],
|
||||
) -> Result[Any]:
|
||||
"""Execute a query asynchronously."""
|
||||
#logger.debug(f"Executing query: {query}")
|
||||
# logger.debug(f"Executing query: {query}, params: {params}")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
start_time = time.perf_counter()
|
||||
if params:
|
||||
result = await session.execute(query, params)
|
||||
else:
|
||||
result = await session.execute(query)
|
||||
result = await session.execute(query, params)
|
||||
end_time = time.perf_counter()
|
||||
elapsed_time = end_time - start_time
|
||||
logger.debug(f"Query executed successfully in {elapsed_time:.2f}s.")
|
||||
return result
|
||||
return result
|
||||
|
||||
@@ -23,7 +23,7 @@ from basic_memory.schemas.delete import (
|
||||
from basic_memory.schemas.request import (
|
||||
SearchNodesRequest,
|
||||
GetEntitiesRequest,
|
||||
CreateRelationsRequest, UpdateEntityRequest,
|
||||
CreateRelationsRequest,
|
||||
)
|
||||
|
||||
# Response models
|
||||
@@ -40,7 +40,8 @@ from basic_memory.schemas.response import (
|
||||
# Discovery and analytics models
|
||||
from basic_memory.schemas.discovery import (
|
||||
EntityTypeList,
|
||||
ObservationCategoryList, TypedEntityList,
|
||||
ObservationCategoryList,
|
||||
TypedEntityList,
|
||||
)
|
||||
|
||||
# For convenient imports, export all models
|
||||
@@ -55,7 +56,6 @@ __all__ = [
|
||||
"SearchNodesRequest",
|
||||
"GetEntitiesRequest",
|
||||
"CreateRelationsRequest",
|
||||
"UpdateEntityRequest",
|
||||
# Responses
|
||||
"SQLAlchemyModel",
|
||||
"ObservationResponse",
|
||||
@@ -69,5 +69,5 @@ __all__ = [
|
||||
# Discovery and Analytics
|
||||
"EntityTypeList",
|
||||
"ObservationCategoryList",
|
||||
"TypedEntityList"
|
||||
"TypedEntityList",
|
||||
]
|
||||
|
||||
@@ -14,14 +14,13 @@ Key Concepts:
|
||||
import mimetypes
|
||||
import re
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Annotated, Dict
|
||||
|
||||
from annotated_types import MinLen, MaxLen
|
||||
from dateparser import parse
|
||||
|
||||
from pydantic import BaseModel, BeforeValidator, Field, model_validator, ValidationError
|
||||
from pydantic import BaseModel, BeforeValidator, Field, model_validator
|
||||
|
||||
from basic_memory.utils import generate_permalink
|
||||
|
||||
@@ -47,44 +46,6 @@ def to_snake_case(name: str) -> str:
|
||||
return s2.lower()
|
||||
|
||||
|
||||
def validate_path_format(path: str) -> str:
|
||||
"""Validate path has the correct format: not empty."""
|
||||
if not path or not isinstance(path, str):
|
||||
raise ValueError("Path must be a non-empty string")
|
||||
|
||||
return path
|
||||
|
||||
|
||||
class ObservationCategory(str, Enum):
|
||||
"""Categories for structuring observations.
|
||||
|
||||
Categories help organize knowledge and make it easier to find later:
|
||||
- tech: Implementation details and technical notes
|
||||
- design: Architecture decisions and patterns
|
||||
- feature: User-facing capabilities
|
||||
- note: General observations (default)
|
||||
- issue: Problems or concerns
|
||||
- todo: Future work items
|
||||
|
||||
Categories are case-insensitive for easier use.
|
||||
"""
|
||||
|
||||
TECH = "tech"
|
||||
DESIGN = "design"
|
||||
FEATURE = "feature"
|
||||
NOTE = "note"
|
||||
ISSUE = "issue"
|
||||
TODO = "todo"
|
||||
|
||||
@classmethod
|
||||
def _missing_(cls, value: str) -> "ObservationCategory":
|
||||
"""Handle case-insensitive lookup."""
|
||||
try:
|
||||
return cls(value.lower())
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def validate_timeframe(timeframe: str) -> str:
|
||||
"""Convert human readable timeframes to a duration relative to the current time."""
|
||||
if not isinstance(timeframe, str):
|
||||
@@ -110,12 +71,9 @@ def validate_timeframe(timeframe: str) -> str:
|
||||
return f"{days}d"
|
||||
|
||||
|
||||
TimeFrame = Annotated[
|
||||
str,
|
||||
BeforeValidator(validate_timeframe)
|
||||
]
|
||||
TimeFrame = Annotated[str, BeforeValidator(validate_timeframe)]
|
||||
|
||||
PathId = Annotated[str, BeforeValidator(validate_path_format)]
|
||||
Permalink = Annotated[str, MinLen(1)]
|
||||
"""Unique identifier in format '{path}/{normalized_name}'."""
|
||||
|
||||
|
||||
@@ -149,14 +107,16 @@ ObservationStr = Annotated[
|
||||
MaxLen(1000), # Keep reasonable length
|
||||
]
|
||||
|
||||
|
||||
class Observation(BaseModel):
|
||||
"""A single observation with category, content, and optional context."""
|
||||
|
||||
category: ObservationCategory
|
||||
category: Optional[str] = None
|
||||
content: ObservationStr
|
||||
tags: Optional[List[str]] = Field(default_factory=list)
|
||||
context: Optional[str] = None
|
||||
|
||||
|
||||
class Relation(BaseModel):
|
||||
"""Represents a directed edge between entities in the knowledge graph.
|
||||
|
||||
@@ -165,8 +125,8 @@ class Relation(BaseModel):
|
||||
or recipient entity.
|
||||
"""
|
||||
|
||||
from_id: PathId
|
||||
to_id: PathId
|
||||
from_id: Permalink
|
||||
to_id: Permalink
|
||||
relation_type: RelationType
|
||||
context: Optional[str] = None
|
||||
|
||||
@@ -181,7 +141,7 @@ class Entity(BaseModel):
|
||||
- Optional relations to other entities
|
||||
- Optional description for high-level overview
|
||||
"""
|
||||
|
||||
|
||||
# private field to override permalink
|
||||
_permalink: Optional[str] = None
|
||||
|
||||
@@ -192,28 +152,27 @@ class Entity(BaseModel):
|
||||
entity_metadata: Optional[Dict] = Field(default=None, description="Optional metadata")
|
||||
content_type: ContentType = Field(
|
||||
description="MIME type of the content (e.g. text/markdown, image/jpeg)",
|
||||
examples=["text/markdown", "image/jpeg"], default="text/markdown"
|
||||
examples=["text/markdown", "image/jpeg"],
|
||||
default="text/markdown",
|
||||
)
|
||||
|
||||
@property
|
||||
def file_path(self):
|
||||
"""Get the file path for this entity based on its permalink."""
|
||||
return f"{self.folder}/{self.title}.md" if self.folder else f"{self.title}.md"
|
||||
|
||||
|
||||
@property
|
||||
def permalink(self) -> PathId:
|
||||
def permalink(self) -> Permalink:
|
||||
"""Get a url friendly path}."""
|
||||
return self._permalink or generate_permalink(self.file_path)
|
||||
|
||||
@model_validator(mode="after")
|
||||
@classmethod
|
||||
def infer_content_type(cls, entity: "Entity") -> Dict | None:
|
||||
"""Infer content_type from file_path if not provided."""
|
||||
if not entity.content_type:
|
||||
path = Path(entity.file_path)
|
||||
def infer_content_type(self) -> "Entity": # pragma: no cover
|
||||
if not self.content_type:
|
||||
path = Path(self.file_path)
|
||||
if not path.exists():
|
||||
return None
|
||||
mime_type, _ = mimetypes.guess_type(path.name)
|
||||
entity.content_type = mime_type or "text/plain"
|
||||
|
||||
return entity
|
||||
self.content_type = "text/plain"
|
||||
else:
|
||||
mime_type, _ = mimetypes.guess_type(path.name)
|
||||
self.content_type = mime_type or "text/plain"
|
||||
return self
|
||||
|
||||
@@ -21,7 +21,7 @@ from typing import List, Annotated
|
||||
from annotated_types import MinLen
|
||||
from pydantic import BaseModel
|
||||
|
||||
from basic_memory.schemas.base import Relation, Observation, PathId
|
||||
from basic_memory.schemas.base import Permalink
|
||||
|
||||
|
||||
class DeleteEntitiesRequest(BaseModel):
|
||||
@@ -34,5 +34,4 @@ class DeleteEntitiesRequest(BaseModel):
|
||||
4. Deletes the corresponding markdown file
|
||||
"""
|
||||
|
||||
permalinks: Annotated[List[PathId], MinLen(1)]
|
||||
|
||||
permalinks: Annotated[List[Permalink], MinLen(1)]
|
||||
|
||||
@@ -8,18 +8,21 @@ from basic_memory.schemas.response import EntityResponse
|
||||
|
||||
class EntityTypeList(BaseModel):
|
||||
"""List of unique entity types in the system."""
|
||||
|
||||
types: List[str]
|
||||
|
||||
|
||||
class ObservationCategoryList(BaseModel):
|
||||
"""List of unique observation categories in the system."""
|
||||
|
||||
categories: List[str]
|
||||
|
||||
|
||||
class TypedEntityList(BaseModel):
|
||||
"""List of entities of a specific type."""
|
||||
|
||||
entity_type: str = Field(..., description="Type of entities in the list")
|
||||
entities: List[EntityResponse]
|
||||
total: int = Field(..., description="Total number of entities")
|
||||
sort_by: Optional[str] = Field(None, description="Field used for sorting")
|
||||
include_related: bool = Field(False, description="Whether related entities are included")
|
||||
include_related: bool = Field(False, description="Whether related entities are included")
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
"""Schemas for memory context."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Any, Optional, Annotated
|
||||
from typing import List, Optional, Annotated, Sequence
|
||||
|
||||
import pydantic
|
||||
from annotated_types import MinLen, MaxLen
|
||||
from pydantic import BaseModel, field_validator, Field, BeforeValidator, TypeAdapter, AnyUrl
|
||||
from pydantic import BaseModel, Field, BeforeValidator, TypeAdapter
|
||||
|
||||
from basic_memory.schemas.search import SearchItemType
|
||||
|
||||
@@ -32,19 +31,20 @@ def normalize_memory_url(url: str) -> str:
|
||||
MemoryUrl = Annotated[
|
||||
str,
|
||||
BeforeValidator(str.strip), # Clean whitespace
|
||||
MinLen(1),
|
||||
MaxLen(2028),
|
||||
MinLen(1),
|
||||
MaxLen(2028),
|
||||
]
|
||||
|
||||
memory_url = TypeAdapter(MemoryUrl)
|
||||
memory_url = TypeAdapter(MemoryUrl)
|
||||
|
||||
def memory_url_path(url: memory_url) -> str:
|
||||
|
||||
def memory_url_path(url: memory_url) -> str: # pyright: ignore
|
||||
"""
|
||||
Returns the uri for a url value by removing the prefix "memory://" from a given MemoryUrl.
|
||||
|
||||
This function processes a given MemoryUrl by removing the "memory://"
|
||||
prefix and returns the resulting string. If the provided url does not
|
||||
begin with "memory://", the function will simply return the input url
|
||||
prefix and returns the resulting string. If the provided url does not
|
||||
begin with "memory://", the function will simply return the input url
|
||||
unchanged.
|
||||
|
||||
:param url: A MemoryUrl object representing the URL with a "memory://" prefix.
|
||||
@@ -55,7 +55,6 @@ def memory_url_path(url: memory_url) -> str:
|
||||
return url.removeprefix("memory://")
|
||||
|
||||
|
||||
|
||||
class EntitySummary(BaseModel):
|
||||
"""Simplified entity representation."""
|
||||
|
||||
@@ -101,14 +100,14 @@ class GraphContext(BaseModel):
|
||||
"""Complete context response."""
|
||||
|
||||
# Direct matches
|
||||
primary_results: List[EntitySummary | RelationSummary | ObservationSummary] = Field(
|
||||
primary_results: Sequence[EntitySummary | RelationSummary | ObservationSummary] = Field(
|
||||
description="results directly matching URI"
|
||||
)
|
||||
|
||||
# Related entities
|
||||
related_results: List[EntitySummary | RelationSummary | ObservationSummary] = Field(
|
||||
related_results: Sequence[EntitySummary | RelationSummary | ObservationSummary] = Field(
|
||||
description="related results"
|
||||
)
|
||||
|
||||
# Context metadata
|
||||
metadata: MemoryMetadata
|
||||
metadata: MemoryMetadata
|
||||
|
||||
@@ -1,23 +1,16 @@
|
||||
"""Request schemas for interacting with the knowledge graph."""
|
||||
|
||||
from typing import List, Optional, Annotated, Dict, Any
|
||||
from typing import List, Optional, Annotated
|
||||
from annotated_types import MaxLen, MinLen
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from basic_memory.schemas.base import (
|
||||
Observation,
|
||||
Entity,
|
||||
Relation,
|
||||
PathId,
|
||||
ObservationCategory,
|
||||
EntityType,
|
||||
Permalink,
|
||||
)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class SearchNodesRequest(BaseModel):
|
||||
"""Search for entities in the knowledge graph.
|
||||
|
||||
@@ -47,7 +40,7 @@ class SearchNodesRequest(BaseModel):
|
||||
"""
|
||||
|
||||
query: Annotated[str, MinLen(1), MaxLen(200)]
|
||||
category: Optional[ObservationCategory] = None
|
||||
category: Optional[str] = None
|
||||
|
||||
|
||||
class GetEntitiesRequest(BaseModel):
|
||||
@@ -58,20 +51,8 @@ class GetEntitiesRequest(BaseModel):
|
||||
discovered through search.
|
||||
"""
|
||||
|
||||
permalinks: Annotated[List[PathId], MinLen(1)]
|
||||
permalinks: Annotated[List[Permalink], MinLen(1)]
|
||||
|
||||
|
||||
class CreateRelationsRequest(BaseModel):
|
||||
relations: List[Relation]
|
||||
|
||||
|
||||
## update
|
||||
|
||||
# TODO remove UpdateEntityRequest
|
||||
class UpdateEntityRequest(BaseModel):
|
||||
"""Request to update an existing entity."""
|
||||
|
||||
title: Optional[str] = None
|
||||
entity_type: Optional[EntityType] = None
|
||||
content: Optional[str] = None
|
||||
entity_metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
@@ -15,7 +15,7 @@ from typing import List, Optional, Dict
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, AliasPath, AliasChoices
|
||||
|
||||
from basic_memory.schemas.base import Relation, PathId, EntityType, ContentType, Observation
|
||||
from basic_memory.schemas.base import Relation, Permalink, EntityType, ContentType, Observation
|
||||
|
||||
|
||||
class SQLAlchemyModel(BaseModel):
|
||||
@@ -28,7 +28,7 @@ class SQLAlchemyModel(BaseModel):
|
||||
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
|
||||
|
||||
class ObservationResponse(Observation, SQLAlchemyModel):
|
||||
"""Schema for observation data returned from the service.
|
||||
|
||||
@@ -59,7 +59,7 @@ class RelationResponse(Relation, SQLAlchemyModel):
|
||||
}
|
||||
"""
|
||||
|
||||
from_id: PathId = Field(
|
||||
from_id: Permalink = Field(
|
||||
# use the permalink from the associated Entity
|
||||
# or the from_id value
|
||||
validation_alias=AliasChoices(
|
||||
@@ -67,25 +67,26 @@ class RelationResponse(Relation, SQLAlchemyModel):
|
||||
"from_id",
|
||||
)
|
||||
)
|
||||
to_id: Optional[PathId] = Field(
|
||||
to_id: Optional[Permalink] = Field( # pyright: ignore
|
||||
# use the permalink from the associated Entity
|
||||
# or the to_id value
|
||||
validation_alias=AliasChoices(
|
||||
AliasPath("to_entity", "permalink"),
|
||||
"to_id",
|
||||
), default=None
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
to_name: Optional[PathId] = Field(
|
||||
to_name: Optional[Permalink] = Field(
|
||||
# use the permalink from the associated Entity
|
||||
# or the to_id value
|
||||
validation_alias=AliasChoices(
|
||||
AliasPath("to_entity", "title"),
|
||||
"to_name",
|
||||
), default=None
|
||||
),
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
|
||||
class EntityResponse(SQLAlchemyModel):
|
||||
"""Complete entity data returned from the service.
|
||||
|
||||
@@ -125,7 +126,7 @@ class EntityResponse(SQLAlchemyModel):
|
||||
}
|
||||
"""
|
||||
|
||||
permalink: PathId
|
||||
permalink: Permalink
|
||||
title: str
|
||||
file_path: str
|
||||
entity_type: EntityType
|
||||
|
||||
@@ -49,8 +49,6 @@ class SearchQuery(BaseModel):
|
||||
@classmethod
|
||||
def validate_date(cls, v: Optional[Union[datetime, str]]) -> Optional[str]:
|
||||
"""Convert datetime to ISO format if needed."""
|
||||
if v is None:
|
||||
return None
|
||||
if isinstance(v, datetime):
|
||||
return v.isoformat()
|
||||
return v
|
||||
@@ -71,12 +69,11 @@ class SearchResult(BaseModel):
|
||||
|
||||
id: int
|
||||
type: SearchItemType
|
||||
score: Optional[float] = None
|
||||
metadata: Optional[dict] = None
|
||||
score: float
|
||||
permalink: str
|
||||
file_path: str
|
||||
|
||||
# Common fields
|
||||
permalink: Optional[str] = None
|
||||
file_path: Optional[str] = None
|
||||
metadata: Optional[dict] = None
|
||||
|
||||
# Type-specific fields
|
||||
entity_id: Optional[int] = None # For observations
|
||||
|
||||
@@ -1,12 +1,7 @@
|
||||
"""Services package."""
|
||||
from .database_service import DatabaseService
|
||||
|
||||
from .service import BaseService
|
||||
from .file_service import FileService
|
||||
from .entity_service import EntityService
|
||||
|
||||
__all__ = [
|
||||
"BaseService",
|
||||
"FileService",
|
||||
"EntityService",
|
||||
"DatabaseService"
|
||||
]
|
||||
__all__ = ["BaseService", "FileService", "EntityService"]
|
||||
|
||||
@@ -50,8 +50,8 @@ class ContextService:
|
||||
|
||||
async def build_context(
|
||||
self,
|
||||
memory_url: MemoryUrl = None,
|
||||
types: List[SearchItemType] = None,
|
||||
memory_url: Optional[MemoryUrl] = None,
|
||||
types: Optional[List[SearchItemType]] = None,
|
||||
depth: int = 1,
|
||||
since: Optional[datetime] = None,
|
||||
max_results: int = 10,
|
||||
@@ -66,9 +66,7 @@ class ContextService:
|
||||
# Pattern matching - use search
|
||||
if "*" in path:
|
||||
logger.debug(f"Pattern search for '{path}'")
|
||||
primary = await self.search_repository.search(
|
||||
permalink_match=path
|
||||
)
|
||||
primary = await self.search_repository.search(permalink_match=path)
|
||||
|
||||
# Direct lookup for exact path
|
||||
else:
|
||||
@@ -120,11 +118,19 @@ class ContextService:
|
||||
- Connected entities
|
||||
- Their observations
|
||||
- Relations that connect them
|
||||
|
||||
Note on depth:
|
||||
Each traversal step requires two depth levels - one to find the relation,
|
||||
and another to follow that relation to an entity. So a max_depth of 4 allows
|
||||
traversal through two entities (relation->entity->relation->entity), while reaching
|
||||
an entity three steps away requires max_depth=6 (relation->entity->relation->entity->relation->entity).
|
||||
"""
|
||||
max_depth = max_depth * 2
|
||||
|
||||
if not type_id_pairs:
|
||||
return []
|
||||
|
||||
logger.debug(f"Finding connected items for {len(type_id_pairs)} with depth {max_depth}")
|
||||
logger.debug(f"Finding connected items for {type_id_pairs} with depth {max_depth}")
|
||||
|
||||
# Build the VALUES clause directly since SQLite doesn't handle parameterized IN well
|
||||
values = ", ".join([f"('{t}', {i})" for t, i in type_id_pairs])
|
||||
@@ -132,7 +138,7 @@ class ContextService:
|
||||
# Parameters for bindings
|
||||
params = {"max_depth": max_depth, "max_results": max_results}
|
||||
if since:
|
||||
params["since_date"] = since.isoformat()
|
||||
params["since_date"] = since.isoformat() # pyright: ignore
|
||||
|
||||
# Build date filter
|
||||
date_filter = "AND base.created_at >= :since_date" if since else ""
|
||||
@@ -140,113 +146,113 @@ class ContextService:
|
||||
related_date_filter = "AND e.created_at >= :since_date" if since else ""
|
||||
|
||||
query = text(f"""
|
||||
WITH RECURSIVE context_graph AS (
|
||||
-- Base case: seed items (unchanged)
|
||||
SELECT
|
||||
id,
|
||||
type,
|
||||
title,
|
||||
permalink,
|
||||
file_path,
|
||||
from_id,
|
||||
to_id,
|
||||
relation_type,
|
||||
content,
|
||||
category,
|
||||
entity_id,
|
||||
0 as depth,
|
||||
id as root_id,
|
||||
created_at,
|
||||
created_at as relation_date,
|
||||
0 as is_incoming
|
||||
FROM search_index base
|
||||
WHERE (base.type, base.id) IN ({values})
|
||||
{date_filter}
|
||||
WITH RECURSIVE context_graph AS (
|
||||
-- Base case: seed items
|
||||
SELECT
|
||||
id,
|
||||
type,
|
||||
title,
|
||||
permalink,
|
||||
file_path,
|
||||
from_id,
|
||||
to_id,
|
||||
relation_type,
|
||||
content,
|
||||
category,
|
||||
entity_id,
|
||||
0 as depth,
|
||||
id as root_id,
|
||||
created_at,
|
||||
created_at as relation_date,
|
||||
0 as is_incoming
|
||||
FROM search_index base
|
||||
WHERE (base.type, base.id) IN ({values})
|
||||
{date_filter}
|
||||
|
||||
UNION -- Changed from UNION ALL
|
||||
UNION ALL -- Allow same paths at different depths
|
||||
|
||||
-- Get relations from current entities
|
||||
SELECT DISTINCT
|
||||
r.id,
|
||||
r.type,
|
||||
r.title,
|
||||
r.permalink,
|
||||
r.file_path,
|
||||
r.from_id,
|
||||
r.to_id,
|
||||
r.relation_type,
|
||||
r.content,
|
||||
r.category,
|
||||
r.entity_id,
|
||||
cg.depth + 1,
|
||||
cg.root_id,
|
||||
r.created_at,
|
||||
r.created_at as relation_date,
|
||||
CASE WHEN r.from_id = cg.id THEN 0 ELSE 1 END as is_incoming
|
||||
FROM context_graph cg
|
||||
JOIN search_index r ON (
|
||||
cg.type = 'entity' AND
|
||||
r.type = 'relation' AND
|
||||
(r.from_id = cg.id OR r.to_id = cg.id)
|
||||
{r1_date_filter}
|
||||
)
|
||||
WHERE cg.depth < :max_depth
|
||||
-- Get relations from current entities
|
||||
SELECT DISTINCT
|
||||
r.id,
|
||||
r.type,
|
||||
r.title,
|
||||
r.permalink,
|
||||
r.file_path,
|
||||
r.from_id,
|
||||
r.to_id,
|
||||
r.relation_type,
|
||||
r.content,
|
||||
r.category,
|
||||
r.entity_id,
|
||||
cg.depth + 1,
|
||||
cg.root_id,
|
||||
r.created_at,
|
||||
r.created_at as relation_date,
|
||||
CASE WHEN r.from_id = cg.id THEN 0 ELSE 1 END as is_incoming
|
||||
FROM context_graph cg
|
||||
JOIN search_index r ON (
|
||||
cg.type = 'entity' AND
|
||||
r.type = 'relation' AND
|
||||
(r.from_id = cg.id OR r.to_id = cg.id)
|
||||
{r1_date_filter}
|
||||
)
|
||||
WHERE cg.depth < :max_depth
|
||||
|
||||
UNION -- Changed from UNION ALL
|
||||
UNION ALL
|
||||
|
||||
-- Get entities connected by relations
|
||||
SELECT DISTINCT
|
||||
e.id,
|
||||
e.type,
|
||||
e.title,
|
||||
e.permalink,
|
||||
e.file_path,
|
||||
e.from_id,
|
||||
e.to_id,
|
||||
e.relation_type,
|
||||
e.content,
|
||||
e.category,
|
||||
e.entity_id,
|
||||
cg.depth,
|
||||
cg.root_id,
|
||||
e.created_at,
|
||||
cg.relation_date,
|
||||
cg.is_incoming
|
||||
FROM context_graph cg
|
||||
JOIN search_index e ON (
|
||||
cg.type = 'relation' AND
|
||||
e.type = 'entity' AND
|
||||
e.id = CASE
|
||||
WHEN cg.from_id = cg.id THEN cg.to_id
|
||||
ELSE cg.from_id
|
||||
END
|
||||
{related_date_filter}
|
||||
)
|
||||
WHERE cg.depth < :max_depth
|
||||
)
|
||||
SELECT DISTINCT
|
||||
type,
|
||||
id,
|
||||
title,
|
||||
permalink,
|
||||
file_path,
|
||||
from_id,
|
||||
to_id,
|
||||
relation_type,
|
||||
content,
|
||||
category,
|
||||
entity_id,
|
||||
MIN(depth) as depth,
|
||||
root_id,
|
||||
created_at
|
||||
FROM context_graph
|
||||
WHERE (type, id) NOT IN ({values})
|
||||
GROUP BY
|
||||
type, id, title, permalink, from_id, to_id,
|
||||
relation_type, category, entity_id,
|
||||
root_id, created_at
|
||||
ORDER BY depth, type, id
|
||||
LIMIT :max_results
|
||||
-- Get entities connected by relations
|
||||
SELECT DISTINCT
|
||||
e.id,
|
||||
e.type,
|
||||
e.title,
|
||||
e.permalink,
|
||||
e.file_path,
|
||||
e.from_id,
|
||||
e.to_id,
|
||||
e.relation_type,
|
||||
e.content,
|
||||
e.category,
|
||||
e.entity_id,
|
||||
cg.depth + 1, -- Increment depth for entities
|
||||
cg.root_id,
|
||||
e.created_at,
|
||||
cg.relation_date,
|
||||
cg.is_incoming
|
||||
FROM context_graph cg
|
||||
JOIN search_index e ON (
|
||||
cg.type = 'relation' AND
|
||||
e.type = 'entity' AND
|
||||
e.id = CASE
|
||||
WHEN cg.is_incoming = 0 THEN cg.to_id -- Fixed entity lookup
|
||||
ELSE cg.from_id
|
||||
END
|
||||
{related_date_filter}
|
||||
)
|
||||
WHERE cg.depth < :max_depth
|
||||
)
|
||||
SELECT DISTINCT
|
||||
type,
|
||||
id,
|
||||
title,
|
||||
permalink,
|
||||
file_path,
|
||||
from_id,
|
||||
to_id,
|
||||
relation_type,
|
||||
content,
|
||||
category,
|
||||
entity_id,
|
||||
MIN(depth) as depth,
|
||||
root_id,
|
||||
created_at
|
||||
FROM context_graph
|
||||
WHERE (type, id) NOT IN ({values})
|
||||
GROUP BY
|
||||
type, id, title, permalink, from_id, to_id,
|
||||
relation_type, category, entity_id,
|
||||
root_id, created_at
|
||||
ORDER BY depth, type, id
|
||||
LIMIT :max_results
|
||||
""")
|
||||
|
||||
result = await self.search_repository.execute_query(query, params=params)
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
"""Service for managing database lifecycle and schema validation."""
|
||||
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple, List
|
||||
|
||||
from alembic.runtime.migration import MigrationContext
|
||||
from alembic.autogenerate import compare_metadata
|
||||
from loguru import logger
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from basic_memory import db
|
||||
from basic_memory.config import ProjectConfig
|
||||
from basic_memory.models import Base
|
||||
|
||||
|
||||
async def check_schema_matches_models(session: AsyncSession) -> Tuple[bool, List[str]]:
|
||||
"""Check if database schema matches SQLAlchemy models.
|
||||
|
||||
Returns:
|
||||
tuple[bool, list[str]]: (matches, list of differences)
|
||||
"""
|
||||
# Get current DB schema via migration context
|
||||
conn = await session.connection()
|
||||
|
||||
def _compare_schemas(connection):
|
||||
context = MigrationContext.configure(connection)
|
||||
return compare_metadata(context, Base.metadata)
|
||||
|
||||
# Run comparison in sync context
|
||||
differences = await conn.run_sync(_compare_schemas)
|
||||
|
||||
if not differences:
|
||||
return True, []
|
||||
|
||||
# Format differences into readable messages
|
||||
diff_messages = []
|
||||
for diff in differences:
|
||||
if diff[0] == 'add_table':
|
||||
diff_messages.append(f"Missing table: {diff[1].name}")
|
||||
elif diff[0] == 'remove_table':
|
||||
diff_messages.append(f"Extra table: {diff[1].name}")
|
||||
elif diff[0] == 'add_column':
|
||||
diff_messages.append(f"Missing column: {diff[3]} in table {diff[2]}")
|
||||
elif diff[0] == 'remove_column':
|
||||
diff_messages.append(f"Extra column: {diff[3]} in table {diff[2]}")
|
||||
elif diff[0] == 'modify_type':
|
||||
diff_messages.append(f"Column type mismatch: {diff[3]} in table {diff[2]}")
|
||||
|
||||
return False, diff_messages
|
||||
|
||||
|
||||
class DatabaseService:
|
||||
"""Manages database lifecycle including schema validation and backups."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: ProjectConfig,
|
||||
db_type: db.DatabaseType = db.DatabaseType.FILESYSTEM,
|
||||
):
|
||||
self.config = config
|
||||
self.db_path = Path(config.database_path)
|
||||
self.db_type = db_type
|
||||
|
||||
async def create_backup(self) -> Optional[Path]:
|
||||
"""Create backup of existing database file.
|
||||
|
||||
Returns:
|
||||
Optional[Path]: Path to backup file if created, None if no DB exists
|
||||
"""
|
||||
if self.db_type == db.DatabaseType.MEMORY:
|
||||
return None # Skip backups for in-memory DB
|
||||
|
||||
if not self.db_path.exists():
|
||||
return None
|
||||
|
||||
# Create backup with timestamp
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
backup_path = self.db_path.with_suffix(f".{timestamp}.backup")
|
||||
|
||||
try:
|
||||
self.db_path.rename(backup_path)
|
||||
logger.info(f"Created database backup: {backup_path}")
|
||||
|
||||
# make a new empty file
|
||||
self.db_path.touch()
|
||||
return backup_path
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create database backup: {e}")
|
||||
return None
|
||||
|
||||
async def initialize_db(self):
|
||||
"""Initialize database with current schema."""
|
||||
logger.info("Initializing database...")
|
||||
|
||||
if self.db_type == db.DatabaseType.FILESYSTEM:
|
||||
await self.create_backup()
|
||||
|
||||
# Drop existing tables if any
|
||||
await db.drop_db()
|
||||
|
||||
# Create tables with current schema
|
||||
await db.get_or_create_db(
|
||||
db_path=self.db_path,
|
||||
db_type=self.db_type
|
||||
)
|
||||
|
||||
logger.info("Database initialized with current schema")
|
||||
|
||||
async def check_db(self) -> bool:
|
||||
"""Check database state and rebuild if schema doesn't match models.
|
||||
|
||||
Returns:
|
||||
bool: True if DB is ready for use, False if initialization failed
|
||||
"""
|
||||
try:
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=self.db_path,
|
||||
db_type=self.db_type
|
||||
)
|
||||
async with db.scoped_session(session_maker) as db_session:
|
||||
# Check actual schema matches
|
||||
matches, differences = await check_schema_matches_models(db_session)
|
||||
if not matches:
|
||||
logger.warning("Database schema does not match models:")
|
||||
for diff in differences:
|
||||
logger.warning(f" {diff}")
|
||||
logger.info("Rebuilding database to match current models...")
|
||||
await self.initialize_db()
|
||||
return True
|
||||
|
||||
logger.info("Database schema matches models")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Database initialization failed: {e}")
|
||||
return False
|
||||
|
||||
async def cleanup_backups(self, keep_count: int = 5):
|
||||
"""Clean up old database backups, keeping the N most recent."""
|
||||
if self.db_type == db.DatabaseType.MEMORY:
|
||||
return # Skip cleanup for in-memory DB
|
||||
|
||||
backup_pattern = "*.backup" # Use relative pattern
|
||||
backups = sorted(
|
||||
self.db_path.parent.glob(backup_pattern),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
# Remove old backups
|
||||
for backup in backups[keep_count:]:
|
||||
try:
|
||||
backup.unlink()
|
||||
logger.debug(f"Removed old backup: {backup}")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to remove backup {backup}: {e}")
|
||||
@@ -1,10 +1,9 @@
|
||||
"""Service for managing entities in the database."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Sequence, List, Optional
|
||||
from typing import Sequence, List, Optional, Tuple, Union
|
||||
|
||||
import frontmatter
|
||||
from frontmatter import Post
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
@@ -14,6 +13,7 @@ from basic_memory.models import Entity as EntityModel, Observation, Relation
|
||||
from basic_memory.repository import ObservationRepository, RelationRepository
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.schemas import Entity as EntitySchema
|
||||
from basic_memory.schemas.base import Permalink
|
||||
from basic_memory.services.exceptions import EntityNotFoundError, EntityCreationError
|
||||
from basic_memory.services import FileService
|
||||
from basic_memory.services import BaseService
|
||||
@@ -42,9 +42,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
self.link_resolver = link_resolver
|
||||
|
||||
async def resolve_permalink(
|
||||
self,
|
||||
file_path: Path,
|
||||
markdown: Optional[EntityMarkdown] = None
|
||||
self, file_path: Permalink | Path, markdown: Optional[EntityMarkdown] = None
|
||||
) -> str:
|
||||
"""Get or generate unique permalink for an entity.
|
||||
|
||||
@@ -54,19 +52,17 @@ class EntityService(BaseService[EntityModel]):
|
||||
3. For existing files, keep current permalink from db
|
||||
4. Generate new unique permalink from file path
|
||||
"""
|
||||
file_path = str(file_path)
|
||||
|
||||
# If markdown has explicit permalink, try to validate it
|
||||
if markdown and markdown.frontmatter.permalink:
|
||||
desired_permalink = markdown.frontmatter.permalink
|
||||
existing = await self.repository.get_by_permalink(desired_permalink)
|
||||
|
||||
# If no conflict or it's our own file, use as is
|
||||
if not existing or existing.file_path == file_path:
|
||||
if not existing or existing.file_path == str(file_path):
|
||||
return desired_permalink
|
||||
|
||||
# For existing files, try to find current permalink
|
||||
existing = await self.repository.get_by_file_path(file_path)
|
||||
existing = await self.repository.get_by_file_path(str(file_path))
|
||||
if existing:
|
||||
return existing.permalink
|
||||
|
||||
@@ -85,12 +81,11 @@ class EntityService(BaseService[EntityModel]):
|
||||
logger.debug(f"creating unique permalink: {permalink}")
|
||||
|
||||
return permalink
|
||||
|
||||
async def create_or_update_entity(self, schema: EntitySchema) -> (EntityModel, bool):
|
||||
"""Create new entity or update existing one.
|
||||
if a new entity is created, the return value is (entity, True)
|
||||
"""
|
||||
|
||||
async def create_or_update_entity(self, schema: EntitySchema) -> Tuple[EntityModel, bool]:
|
||||
"""Create new entity or update existing one.
|
||||
Returns: (entity, is_new) where is_new is True if a new entity was created
|
||||
"""
|
||||
logger.debug(f"Creating or updating entity: {schema}")
|
||||
|
||||
# Try to find existing entity using smart resolution
|
||||
@@ -107,7 +102,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
"""Create a new entity and write to filesystem."""
|
||||
logger.debug(f"Creating entity: {schema.permalink}")
|
||||
|
||||
# get file path
|
||||
# Get file path and ensure it's a Path object
|
||||
file_path = Path(schema.file_path)
|
||||
|
||||
if await self.file_service.exists(file_path):
|
||||
@@ -127,11 +122,9 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# parse entity from file
|
||||
entity_markdown = await self.entity_parser.parse_file(file_path)
|
||||
|
||||
|
||||
# create entity
|
||||
created_entity = await self.create_entity_from_markdown(
|
||||
file_path, entity_markdown
|
||||
)
|
||||
await self.create_entity_from_markdown(file_path, entity_markdown)
|
||||
|
||||
# add relations
|
||||
entity = await self.update_entity_relations(file_path, entity_markdown)
|
||||
@@ -139,12 +132,11 @@ class EntityService(BaseService[EntityModel]):
|
||||
# Set final checksum to mark complete
|
||||
return await self.repository.update(entity.id, {"checksum": checksum})
|
||||
|
||||
|
||||
async def update_entity(self, entity: EntityModel, schema: EntitySchema) -> EntityModel:
|
||||
"""Update an entity's content and metadata."""
|
||||
logger.debug(f"Updating entity with permalink: {entity.permalink}")
|
||||
|
||||
# get file path
|
||||
# Convert file path string to Path
|
||||
file_path = Path(entity.file_path)
|
||||
|
||||
post = await schema_to_markdown(schema)
|
||||
@@ -157,9 +149,7 @@ class EntityService(BaseService[EntityModel]):
|
||||
entity_markdown = await self.entity_parser.parse_file(file_path)
|
||||
|
||||
# update entity in db
|
||||
entity = await self.update_entity_and_observations(
|
||||
file_path, entity_markdown
|
||||
)
|
||||
entity = await self.update_entity_and_observations(file_path, entity_markdown)
|
||||
|
||||
# add relations
|
||||
await self.update_entity_relations(file_path, entity_markdown)
|
||||
@@ -187,10 +177,6 @@ class EntityService(BaseService[EntityModel]):
|
||||
logger.info(f"Entity not found: {permalink}")
|
||||
return True # Already deleted
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete entity: {e}")
|
||||
raise
|
||||
|
||||
async def get_by_permalink(self, permalink: str) -> EntityModel:
|
||||
"""Get entity by type and name combination."""
|
||||
logger.debug(f"Getting entity by permalink: {permalink}")
|
||||
@@ -199,32 +185,14 @@ class EntityService(BaseService[EntityModel]):
|
||||
raise EntityNotFoundError(f"Entity not found: {permalink}")
|
||||
return db_entity
|
||||
|
||||
async def get_all(self) -> Sequence[EntityModel]:
|
||||
"""Get all entities."""
|
||||
return await self.repository.find_all()
|
||||
|
||||
async def get_entity_types(self) -> List[str]:
|
||||
"""Get list of all distinct entity types in the system."""
|
||||
logger.debug("Getting all distinct entity types")
|
||||
return await self.repository.get_entity_types()
|
||||
|
||||
async def list_entities(
|
||||
self,
|
||||
entity_type: Optional[str] = None,
|
||||
sort_by: Optional[str] = "updated_at",
|
||||
include_related: bool = False,
|
||||
) -> Sequence[EntityModel]:
|
||||
"""List entities with optional filtering and sorting."""
|
||||
logger.debug(f"Listing entities: type={entity_type} sort={sort_by}")
|
||||
return await self.repository.list_entities(entity_type=entity_type, sort_by=sort_by)
|
||||
|
||||
async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
|
||||
"""Get specific nodes and their relationships."""
|
||||
logger.debug(f"Getting entities permalinks: {permalinks}")
|
||||
return await self.repository.find_by_permalinks(permalinks)
|
||||
|
||||
async def delete_entity_by_file_path(self, file_path):
|
||||
await self.repository.delete_by_file_path(file_path)
|
||||
async def delete_entity_by_file_path(self, file_path: Union[str, Path]) -> None:
|
||||
"""Delete entity by file path."""
|
||||
await self.repository.delete_by_file_path(str(file_path))
|
||||
|
||||
async def create_entity_from_markdown(
|
||||
self, file_path: Path, markdown: EntityMarkdown
|
||||
@@ -234,15 +202,15 @@ class EntityService(BaseService[EntityModel]):
|
||||
Creates the entity with null checksum to indicate sync not complete.
|
||||
Relations will be added in second pass.
|
||||
"""
|
||||
logger.debug(f"Creating entity: {markdown.frontmatter.title}")
|
||||
logger.debug(f"Creating entity: {markdown.frontmatter.title}")
|
||||
model = entity_model_from_markdown(file_path, markdown)
|
||||
|
||||
# Mark as incomplete sync
|
||||
# Mark as incomplete because we still need to add relations
|
||||
model.checksum = None
|
||||
return await self.add(model)
|
||||
return await self.repository.add(model)
|
||||
|
||||
async def update_entity_and_observations(
|
||||
self, file_path: Path | str, markdown: EntityMarkdown
|
||||
self, file_path: Path, markdown: EntityMarkdown
|
||||
) -> EntityModel:
|
||||
"""Update entity fields and observations.
|
||||
|
||||
@@ -250,11 +218,8 @@ class EntityService(BaseService[EntityModel]):
|
||||
to indicate sync not complete.
|
||||
"""
|
||||
logger.debug(f"Updating entity and observations: {file_path}")
|
||||
file_path = str(file_path)
|
||||
|
||||
db_entity = await self.repository.get_by_file_path(file_path)
|
||||
if not db_entity:
|
||||
raise EntityNotFoundError(f"Entity not found: {file_path}")
|
||||
db_entity = await self.repository.get_by_file_path(str(file_path))
|
||||
|
||||
# Clear observations for entity
|
||||
await self.observation_repository.delete_by_fields(entity_id=db_entity.id)
|
||||
@@ -277,9 +242,8 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
# checksum value is None == not finished with sync
|
||||
db_entity.checksum = None
|
||||
|
||||
|
||||
# update entity
|
||||
# checksum value is None == not finished with sync
|
||||
return await self.repository.update(
|
||||
db_entity.id,
|
||||
db_entity,
|
||||
@@ -287,14 +251,13 @@ class EntityService(BaseService[EntityModel]):
|
||||
|
||||
async def update_entity_relations(
|
||||
self,
|
||||
file_path: Path | str,
|
||||
file_path: Path,
|
||||
markdown: EntityMarkdown,
|
||||
) -> EntityModel:
|
||||
"""Update relations for entity"""
|
||||
logger.debug(f"Updating relations for entity: {file_path}")
|
||||
|
||||
file_path = str(file_path)
|
||||
db_entity = await self.repository.get_by_file_path(file_path)
|
||||
db_entity = await self.repository.get_by_file_path(str(file_path))
|
||||
|
||||
# Clear existing relations first
|
||||
await self.relation_repository.delete_outgoing_relations_from_entity(db_entity.id)
|
||||
@@ -328,4 +291,4 @@ class EntityService(BaseService[EntityModel]):
|
||||
)
|
||||
continue
|
||||
|
||||
return await self.repository.get_by_file_path(file_path)
|
||||
return await self.repository.get_by_file_path(str(file_path))
|
||||
|
||||
@@ -9,6 +9,7 @@ class EntityNotFoundError(Exception):
|
||||
|
||||
pass
|
||||
|
||||
|
||||
class EntityCreationError(Exception):
|
||||
"""Raised when an entity cannot be created"""
|
||||
|
||||
|
||||
@@ -1,20 +1,23 @@
|
||||
"""Service for file operations with checksum tracking."""
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Optional, Tuple
|
||||
from typing import Tuple, Union
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory import file_utils
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.markdown.utils import entity_model_to_markdown
|
||||
from basic_memory.models import Entity as EntityModel
|
||||
from basic_memory.services.exceptions import FileOperationError
|
||||
from basic_memory.schemas import Entity as EntitySchema
|
||||
from basic_memory.services.exceptions import FileOperationError
|
||||
|
||||
|
||||
class FileService:
|
||||
"""
|
||||
Service for handling file operations.
|
||||
"""Service for handling file operations.
|
||||
|
||||
All paths are handled as Path objects internally. Strings are converted to
|
||||
Path objects when passed in. Relative paths are assumed to be relative to
|
||||
base_path.
|
||||
|
||||
Features:
|
||||
- Consistent file writing with checksums
|
||||
@@ -28,105 +31,66 @@ class FileService:
|
||||
base_path: Path,
|
||||
markdown_processor: MarkdownProcessor,
|
||||
):
|
||||
self.base_path = base_path
|
||||
self.base_path = base_path.resolve() # Get absolute path
|
||||
self.markdown_processor = markdown_processor
|
||||
|
||||
def get_entity_path(self, entity: EntityModel| EntitySchema) -> Path:
|
||||
"""Generate absolute filesystem path for entity."""
|
||||
return self.base_path / f"{entity.file_path}"
|
||||
|
||||
async def write_entity_file(
|
||||
self,
|
||||
entity: EntityModel,
|
||||
content: Optional[str] = None,
|
||||
expected_checksum: Optional[str] = None,
|
||||
) -> Tuple[Path, str]:
|
||||
"""Write entity to filesystem and return path and checksum.
|
||||
|
||||
Uses read->modify->write pattern:
|
||||
1. Read existing file if it exists
|
||||
2. Update with new content if provided
|
||||
3. Write back atomically
|
||||
def get_entity_path(self, entity: Union[EntityModel, EntitySchema]) -> Path:
|
||||
"""Generate absolute filesystem path for entity.
|
||||
|
||||
Args:
|
||||
entity: Entity model to write
|
||||
content: Optional new content (preserves existing if None)
|
||||
expected_checksum: Optional checksum to verify file hasn't changed
|
||||
entity: Entity model or schema with file_path attribute
|
||||
|
||||
Returns:
|
||||
Tuple of (file path, new checksum)
|
||||
|
||||
Raises:
|
||||
FileOperationError: If write fails
|
||||
Absolute Path to the entity file
|
||||
"""
|
||||
try:
|
||||
path = self.get_entity_path(entity)
|
||||
|
||||
# Read current state if file exists
|
||||
if path.exists():
|
||||
# read the existing file
|
||||
existing_markdown = await self.markdown_processor.read_file(path)
|
||||
|
||||
# if content is supplied use it or existing content
|
||||
content=content or existing_markdown.content
|
||||
|
||||
# Create new file structure with provided content
|
||||
markdown = entity_model_to_markdown(entity, content=content)
|
||||
|
||||
# Write back atomically
|
||||
checksum = await self.markdown_processor.write_file(
|
||||
path=path, markdown=markdown, expected_checksum=expected_checksum
|
||||
)
|
||||
|
||||
return path, checksum
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to write entity file: {e}")
|
||||
raise FileOperationError(f"Failed to write entity file: {e}")
|
||||
return self.base_path / entity.file_path
|
||||
|
||||
async def read_entity_content(self, entity: EntityModel) -> str:
|
||||
"""Get entity's content without frontmatter or structured sections (used to index for search)
|
||||
"""Get entity's content without frontmatter or structured sections.
|
||||
|
||||
Used to index for search. Returns raw content without frontmatter,
|
||||
observations, or relations.
|
||||
|
||||
Args:
|
||||
entity: Entity to read content for
|
||||
|
||||
Returns:
|
||||
Raw content without frontmatter, observations, or relations
|
||||
|
||||
Raises:
|
||||
FileOperationError: If entity file doesn't exist
|
||||
Raw content string without metadata sections
|
||||
"""
|
||||
logger.debug(f"Reading entity with permalink: {entity.permalink}")
|
||||
|
||||
try:
|
||||
file_path = self.get_entity_path(entity)
|
||||
markdown = await self.markdown_processor.read_file(file_path)
|
||||
return markdown.content or ""
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read entity content: {e}")
|
||||
raise FileOperationError(f"Failed to read entity content: {e}")
|
||||
file_path = self.get_entity_path(entity)
|
||||
markdown = await self.markdown_processor.read_file(file_path)
|
||||
return markdown.content or ""
|
||||
|
||||
async def delete_entity_file(self, entity: EntityModel) -> None:
|
||||
"""Delete entity file from filesystem."""
|
||||
try:
|
||||
path = self.get_entity_path(entity)
|
||||
await self.delete_file(path)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete entity file: {e}")
|
||||
raise FileOperationError(f"Failed to delete entity file: {e}")
|
||||
|
||||
async def exists(self, path: Path) -> bool:
|
||||
"""
|
||||
Check if file exists at the provided path. If path is relative, it is assumed to be relative to base_path.
|
||||
"""Delete entity file from filesystem.
|
||||
|
||||
Args:
|
||||
path: Path to check
|
||||
entity: Entity model whose file should be deleted
|
||||
|
||||
Raises:
|
||||
FileOperationError: If deletion fails
|
||||
"""
|
||||
path = self.get_entity_path(entity)
|
||||
await self.delete_file(path)
|
||||
|
||||
async def exists(self, path: Union[Path, str]) -> bool:
|
||||
"""Check if file exists at the provided path.
|
||||
|
||||
If path is relative, it is assumed to be relative to base_path.
|
||||
|
||||
Args:
|
||||
path: Path to check (Path object or string)
|
||||
|
||||
Returns:
|
||||
True if file exists, False otherwise
|
||||
|
||||
Raises:
|
||||
FileOperationError: If check fails
|
||||
"""
|
||||
try:
|
||||
path = Path(path)
|
||||
if path.is_absolute():
|
||||
return path.exists()
|
||||
else:
|
||||
@@ -135,12 +99,14 @@ class FileService:
|
||||
logger.error(f"Failed to check file existence {path}: {e}")
|
||||
raise FileOperationError(f"Failed to check file existence: {e}")
|
||||
|
||||
async def write_file(self, path: Path, content: str) -> str:
|
||||
"""
|
||||
Write content to file and return checksum.
|
||||
async def write_file(self, path: Union[Path, str], content: str) -> str:
|
||||
"""Write content to file and return checksum.
|
||||
|
||||
Handles both absolute and relative paths. Relative paths are resolved
|
||||
against base_path.
|
||||
|
||||
Args:
|
||||
path: Path where to write
|
||||
path: Where to write (Path object or string)
|
||||
content: Content to write
|
||||
|
||||
Returns:
|
||||
@@ -149,30 +115,33 @@ class FileService:
|
||||
Raises:
|
||||
FileOperationError: If write fails
|
||||
"""
|
||||
|
||||
path = path if path.is_absolute() else self.base_path / path
|
||||
path = Path(path)
|
||||
full_path = path if path.is_absolute() else self.base_path / path
|
||||
|
||||
try:
|
||||
# Ensure parent directory exists
|
||||
await file_utils.ensure_directory(path.parent)
|
||||
await file_utils.ensure_directory(full_path.parent)
|
||||
|
||||
# Write content atomically
|
||||
await file_utils.write_file_atomic(path, content)
|
||||
await file_utils.write_file_atomic(full_path, content)
|
||||
|
||||
# Compute and return checksum
|
||||
checksum = await file_utils.compute_checksum(content)
|
||||
logger.debug(f"wrote file: {path}, checksum: {checksum} content: \n{content}")
|
||||
logger.debug(f"wrote file: {full_path}, checksum: {checksum}")
|
||||
return checksum
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to write file {path}: {e}")
|
||||
logger.error(f"Failed to write file {full_path}: {e}")
|
||||
raise FileOperationError(f"Failed to write file: {e}")
|
||||
|
||||
async def read_file(self, path: Path) -> Tuple[str, str]:
|
||||
"""
|
||||
Read file and compute checksum.
|
||||
async def read_file(self, path: Union[Path, str]) -> Tuple[str, str]:
|
||||
"""Read file and compute checksum.
|
||||
|
||||
Handles both absolute and relative paths. Relative paths are resolved
|
||||
against base_path.
|
||||
|
||||
Args:
|
||||
path: Path to read
|
||||
path: Path to read (Path object or string)
|
||||
|
||||
Returns:
|
||||
Tuple of (content, checksum)
|
||||
@@ -180,33 +149,28 @@ class FileService:
|
||||
Raises:
|
||||
FileOperationError: If read fails
|
||||
"""
|
||||
path = path if path.is_absolute() else self.base_path / path
|
||||
path = Path(path)
|
||||
full_path = path if path.is_absolute() else self.base_path / path
|
||||
|
||||
try:
|
||||
content = path.read_text()
|
||||
checksum = await file_utils.compute_checksum(content)
|
||||
logger.debug(f"read file: {path}, checksum: {checksum}")
|
||||
logger.debug(f"read file: {full_path}, checksum: {checksum}")
|
||||
return content, checksum
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read file {path}: {e}")
|
||||
logger.error(f"Failed to read file {full_path}: {e}")
|
||||
raise FileOperationError(f"Failed to read file: {e}")
|
||||
|
||||
async def delete_file(self, path: Path) -> None:
|
||||
"""
|
||||
Delete file if it exists.
|
||||
async def delete_file(self, path: Union[Path, str]) -> None:
|
||||
"""Delete file if it exists.
|
||||
|
||||
Handles both absolute and relative paths. Relative paths are resolved
|
||||
against base_path.
|
||||
|
||||
Args:
|
||||
path: Path to delete
|
||||
|
||||
Raises:
|
||||
FileOperationError: If deletion fails
|
||||
path: Path to delete (Path object or string)
|
||||
"""
|
||||
path = path if path.is_absolute() else self.base_path / path
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to delete file {path}: {e}")
|
||||
raise FileOperationError(f"Failed to delete file: {e}")
|
||||
|
||||
def path(self, path_string: str, absolute: bool = False):
|
||||
return Path( self.base_path / path_string ) if absolute else Path(path_string).relative_to(self.base_path)
|
||||
path = Path(path)
|
||||
full_path = path if path.is_absolute() else self.base_path / path
|
||||
full_path.unlink(missing_ok=True)
|
||||
|
||||
@@ -5,9 +5,10 @@ from typing import Optional, Tuple, List
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.repository.entity_repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.services.search_service import SearchService
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
|
||||
from basic_memory.schemas.search import SearchQuery, SearchItemType
|
||||
|
||||
|
||||
class LinkResolver:
|
||||
@@ -45,18 +46,19 @@ class LinkResolver:
|
||||
return entity
|
||||
|
||||
if use_search:
|
||||
|
||||
# 3. Fall back to search for fuzzy matching on title if specified
|
||||
results = await self.search_service.search(
|
||||
query=SearchQuery(title=clean_text, types=[SearchItemType.ENTITY]),
|
||||
)
|
||||
|
||||
|
||||
if results:
|
||||
# Look for best match
|
||||
best_match = self._select_best_match(clean_text, results)
|
||||
logger.debug(f"Selected best match from {len(results)} results: {best_match.permalink}")
|
||||
logger.debug(
|
||||
f"Selected best match from {len(results)} results: {best_match.permalink}"
|
||||
)
|
||||
return await self.entity_repository.get_by_permalink(best_match.permalink)
|
||||
|
||||
|
||||
# if we couldn't find anything then return None
|
||||
return None
|
||||
|
||||
@@ -85,7 +87,7 @@ class LinkResolver:
|
||||
|
||||
return text, alias
|
||||
|
||||
def _select_best_match(self, search_text: str, results: List[SearchResult]) -> Entity:
|
||||
def _select_best_match(self, search_text: str, results: List[SearchIndexRow]) -> Entity:
|
||||
"""Select best match from search results.
|
||||
|
||||
Uses multiple criteria:
|
||||
@@ -93,9 +95,6 @@ class LinkResolver:
|
||||
2. Word matches in path
|
||||
3. Overall search score
|
||||
"""
|
||||
if not results:
|
||||
raise ValueError("Cannot select from empty results")
|
||||
|
||||
# Get search terms for matching
|
||||
terms = search_text.lower().split()
|
||||
|
||||
@@ -104,6 +103,7 @@ class LinkResolver:
|
||||
for result in results:
|
||||
# Start with base score (lower is better)
|
||||
score = result.score
|
||||
assert score is not None
|
||||
|
||||
# Parse path components
|
||||
path_parts = result.permalink.lower().split("/")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""Service for search operations."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Set
|
||||
|
||||
from fastapi import BackgroundTasks
|
||||
@@ -8,9 +9,8 @@ from loguru import logger
|
||||
from basic_memory.models import Entity
|
||||
from basic_memory.repository import EntityRepository
|
||||
from basic_memory.repository.search_repository import SearchRepository, SearchIndexRow
|
||||
from basic_memory.schemas.search import SearchQuery, SearchResult, SearchItemType
|
||||
from basic_memory.schemas.search import SearchQuery, SearchItemType
|
||||
from basic_memory.services import FileService
|
||||
from basic_memory.services.exceptions import FileOperationError
|
||||
|
||||
|
||||
class SearchService:
|
||||
@@ -51,9 +51,7 @@ class SearchService:
|
||||
|
||||
logger.info("Reindex complete")
|
||||
|
||||
async def search(
|
||||
self, query: SearchQuery, context: Optional[List[str]] = None
|
||||
) -> List[SearchResult]:
|
||||
async def search(self, query: SearchQuery) -> List[SearchIndexRow]:
|
||||
"""Search across all indexed content.
|
||||
|
||||
Supports three modes:
|
||||
@@ -67,6 +65,16 @@ class SearchService:
|
||||
|
||||
logger.debug(f"Searching with query: {query}")
|
||||
|
||||
after_date = (
|
||||
(
|
||||
query.after_date
|
||||
if isinstance(query.after_date, datetime)
|
||||
else datetime.fromisoformat(query.after_date)
|
||||
)
|
||||
if query.after_date
|
||||
else None
|
||||
)
|
||||
|
||||
# permalink search
|
||||
results = await self.repository.search(
|
||||
search_text=query.text,
|
||||
@@ -75,12 +83,13 @@ class SearchService:
|
||||
title=query.title,
|
||||
types=query.types,
|
||||
entity_types=query.entity_types,
|
||||
after_date=query.after_date,
|
||||
after_date=after_date,
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
def _generate_variants(self, text: str) -> Set[str]:
|
||||
@staticmethod
|
||||
def _generate_variants(text: str) -> Set[str]:
|
||||
"""Generate text variants for better fuzzy matching.
|
||||
|
||||
Creates variations of the text to improve match chances:
|
||||
@@ -140,7 +149,6 @@ class SearchService:
|
||||
title_variants = self._generate_variants(entity.title)
|
||||
content_parts.extend(title_variants)
|
||||
|
||||
# TODO should we do something to content on indexing?
|
||||
content = await self.file_service.read_entity_content(entity)
|
||||
if content:
|
||||
content_parts.append(content)
|
||||
@@ -162,8 +170,8 @@ class SearchService:
|
||||
metadata={
|
||||
"entity_type": entity.entity_type,
|
||||
},
|
||||
created_at=entity.created_at.isoformat(),
|
||||
updated_at=entity.updated_at.isoformat(),
|
||||
created_at=entity.created_at,
|
||||
updated_at=entity.updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -183,8 +191,8 @@ class SearchService:
|
||||
metadata={
|
||||
"tags": obs.tags,
|
||||
},
|
||||
created_at=entity.created_at.isoformat(),
|
||||
updated_at=entity.updated_at.isoformat(),
|
||||
created_at=entity.created_at,
|
||||
updated_at=entity.updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
@@ -201,15 +209,14 @@ class SearchService:
|
||||
SearchIndexRow(
|
||||
id=rel.id,
|
||||
title=relation_title,
|
||||
content=rel.context or "",
|
||||
permalink=rel.permalink,
|
||||
file_path=entity.file_path,
|
||||
type=SearchItemType.RELATION.value,
|
||||
from_id=rel.from_id,
|
||||
to_id=rel.to_id,
|
||||
relation_type=rel.relation_type,
|
||||
created_at=entity.created_at.isoformat(),
|
||||
updated_at=entity.updated_at.isoformat(),
|
||||
created_at=entity.created_at,
|
||||
updated_at=entity.updated_at,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -1,36 +1,15 @@
|
||||
"""Base service class."""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import TypeVar, Generic, List, Sequence
|
||||
from typing import TypeVar, Generic
|
||||
|
||||
from basic_memory.models import Base
|
||||
from basic_memory.repository.repository import Repository
|
||||
|
||||
T = TypeVar("T", bound=Base)
|
||||
R = TypeVar("R", bound=Repository)
|
||||
|
||||
|
||||
class BaseService(Generic[T]):
|
||||
"""Base service that takes a repository."""
|
||||
|
||||
def __init__(self, repository: R):
|
||||
def __init__(self, repository):
|
||||
"""Initialize service with repository."""
|
||||
self.repository = repository
|
||||
|
||||
async def add(self, model: T) -> T:
|
||||
"""Add model to repository."""
|
||||
return await self.repository.add(model)
|
||||
|
||||
async def add_all(self, models: List[T]) -> Sequence[T]:
|
||||
"""Add a List of models to repository."""
|
||||
return await self.repository.add_all(models)
|
||||
|
||||
async def get_modified_since(self, since: datetime) -> Sequence[T]:
|
||||
"""Get all items modified since the given timestamp.
|
||||
|
||||
Args:
|
||||
since: Datetime to search from
|
||||
|
||||
Returns:
|
||||
Sequence of items modified since the timestamp
|
||||
"""
|
||||
return await self.repository.find_modified_since(since)
|
||||
@@ -1,5 +1,5 @@
|
||||
from .file_change_scanner import FileChangeScanner
|
||||
from .sync_service import SyncService
|
||||
from .watch_service import WatchService
|
||||
|
||||
__all__ = ["SyncService", "FileChangeScanner"]
|
||||
|
||||
__all__ = ["SyncService", "FileChangeScanner", "WatchService"]
|
||||
|
||||
@@ -69,11 +69,7 @@ class FileChangeScanner:
|
||||
rel_path = str(path.relative_to(directory))
|
||||
content = path.read_text()
|
||||
checksum = await compute_checksum(content)
|
||||
|
||||
if checksum: # Only store valid checksums
|
||||
result.files[rel_path] = checksum
|
||||
else:
|
||||
result.errors[rel_path] = "Failed to compute checksum"
|
||||
result.files[rel_path] = checksum
|
||||
|
||||
except Exception as e:
|
||||
rel_path = str(path.relative_to(directory))
|
||||
@@ -134,7 +130,7 @@ class FileChangeScanner:
|
||||
logger.debug(f" Moved: {len(report.moves)}")
|
||||
logger.debug(f" Deleted: {len(report.deleted)}")
|
||||
|
||||
if scan_result.errors:
|
||||
if scan_result.errors: # pragma: no cover
|
||||
logger.warning("Files skipped due to errors:")
|
||||
for file_path, error in scan_result.errors.items():
|
||||
logger.warning(f" {file_path}: {error}")
|
||||
@@ -151,7 +147,7 @@ class FileChangeScanner:
|
||||
"""
|
||||
return {
|
||||
r.file_path: FileState(
|
||||
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum
|
||||
file_path=r.file_path, permalink=r.permalink, checksum=r.checksum or ""
|
||||
)
|
||||
for r in db_records
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from basic_memory import file_utils
|
||||
from basic_memory.markdown import EntityParser, EntityMarkdown
|
||||
@@ -58,9 +59,6 @@ class SyncService:
|
||||
for permalink in permalinks:
|
||||
await self.search_service.delete_by_permalink(permalink)
|
||||
|
||||
else:
|
||||
logger.debug(f"No entity found to delete: {file_path}")
|
||||
|
||||
async def sync(self, directory: Path) -> SyncReport:
|
||||
"""Sync knowledge files with database."""
|
||||
changes = await self.scanner.find_knowledge_changes(directory)
|
||||
@@ -76,69 +74,63 @@ class SyncService:
|
||||
entity.id, {"file_path": new_path, "checksum": changes.checksums[new_path]}
|
||||
)
|
||||
# update search index
|
||||
await self.search_service.index_entity(updated)
|
||||
if updated:
|
||||
await self.search_service.index_entity(updated)
|
||||
|
||||
# Handle deletions next
|
||||
# remove rows from db for files no longer present
|
||||
for file_path in changes.deleted:
|
||||
await self.handle_entity_deletion(file_path)
|
||||
for path in changes.deleted:
|
||||
await self.handle_entity_deletion(path)
|
||||
|
||||
# Parse files that need updating
|
||||
parsed_entities: Dict[str, EntityMarkdown] = {}
|
||||
|
||||
for file_path in [*changes.new, *changes.modified]:
|
||||
entity_markdown = await self.entity_parser.parse_file(directory / file_path)
|
||||
parsed_entities[file_path] = entity_markdown
|
||||
for path in [*changes.new, *changes.modified]:
|
||||
entity_markdown = await self.entity_parser.parse_file(directory / path)
|
||||
parsed_entities[path] = entity_markdown
|
||||
|
||||
# First pass: Create/update entities
|
||||
# entities will have a null checksum to indicate they are not complete
|
||||
for file_path, entity_markdown in parsed_entities.items():
|
||||
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
# Get unique permalink and update markdown if needed
|
||||
permalink = await self.entity_service.resolve_permalink(
|
||||
file_path,
|
||||
markdown=entity_markdown
|
||||
Path(path), markdown=entity_markdown
|
||||
)
|
||||
|
||||
if permalink != entity_markdown.frontmatter.permalink:
|
||||
# Add/update permalink in frontmatter
|
||||
logger.info(f"Adding permalink '{permalink}' to file: {file_path}")
|
||||
logger.info(f"Adding permalink '{permalink}' to file: {path}")
|
||||
|
||||
# update markdown
|
||||
entity_markdown.frontmatter.metadata["permalink"] = permalink
|
||||
|
||||
|
||||
# update file frontmatter
|
||||
updated_checksum = await file_utils.update_frontmatter(
|
||||
directory / file_path,
|
||||
{"permalink": permalink}
|
||||
directory / path, {"permalink": permalink}
|
||||
)
|
||||
|
||||
# Update checksum in changes report since file was modified
|
||||
changes.checksums[file_path] = updated_checksum
|
||||
|
||||
changes.checksums[path] = updated_checksum
|
||||
|
||||
# if the file is new, create an entity
|
||||
if file_path in changes.new:
|
||||
if path in changes.new:
|
||||
# Create entity with final permalink
|
||||
logger.debug(f"Creating new entity_markdown: {file_path}")
|
||||
await self.entity_service.create_entity_from_markdown(
|
||||
file_path, entity_markdown
|
||||
)
|
||||
logger.debug(f"Creating new entity_markdown: {path}")
|
||||
await self.entity_service.create_entity_from_markdown(Path(path), entity_markdown)
|
||||
# otherwise we need to update the entity and observations
|
||||
else:
|
||||
logger.debug(f"Updating entity_markdown: {file_path}")
|
||||
logger.debug(f"Updating entity_markdown: {path}")
|
||||
await self.entity_service.update_entity_and_observations(
|
||||
file_path, entity_markdown
|
||||
Path(path), entity_markdown
|
||||
)
|
||||
|
||||
# Second pass
|
||||
for file_path, entity_markdown in parsed_entities.items():
|
||||
logger.debug(f"Updating relations for: {file_path}")
|
||||
for path, entity_markdown in parsed_entities.items():
|
||||
logger.debug(f"Updating relations for: {path}")
|
||||
|
||||
# Process relations
|
||||
checksum = changes.checksums[file_path]
|
||||
entity = await self.entity_service.update_entity_relations(
|
||||
file_path, entity_markdown
|
||||
)
|
||||
checksum = changes.checksums[path]
|
||||
entity = await self.entity_service.update_entity_relations(Path(path), entity_markdown)
|
||||
|
||||
# add to search index
|
||||
await self.search_service.index_entity(entity)
|
||||
@@ -152,14 +144,22 @@ class SyncService:
|
||||
target_entity = await self.entity_service.link_resolver.resolve_link(relation.to_name)
|
||||
# check we found a link that is not the source
|
||||
if target_entity and target_entity.id != relation.from_id:
|
||||
logger.debug(f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}")
|
||||
await self.relation_repository.update(relation.id, {
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title # Update to actual title
|
||||
})
|
||||
logger.debug(
|
||||
f"Resolved forward reference: {relation.to_name} -> {target_entity.permalink}"
|
||||
)
|
||||
|
||||
try:
|
||||
await self.relation_repository.update(
|
||||
relation.id,
|
||||
{
|
||||
"to_id": target_entity.id,
|
||||
"to_name": target_entity.title, # Update to actual title
|
||||
},
|
||||
)
|
||||
except IntegrityError:
|
||||
logger.debug(f"Ignoring duplicate relation {relation}")
|
||||
|
||||
# update search index
|
||||
await self.search_service.index_entity(target_entity)
|
||||
|
||||
|
||||
return changes
|
||||
|
||||
@@ -1,45 +1,13 @@
|
||||
"""Types and utilities for file sync."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Set, Dict, Optional
|
||||
from watchfiles import Change
|
||||
from basic_memory.services.file_service import FileService
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileChange:
|
||||
"""A change to a file detected by the watch service.
|
||||
|
||||
Attributes:
|
||||
change_type: Type of change (added, modified, deleted)
|
||||
path: Path to the file
|
||||
checksum: File checksum (None for deleted files)
|
||||
"""
|
||||
change_type: Change
|
||||
path: str
|
||||
checksum: Optional[str] = None
|
||||
|
||||
@classmethod
|
||||
async def from_path(cls, path: str, change_type: Change, file_service: FileService) -> "FileChange":
|
||||
"""Create FileChange from a path, computing checksum if file exists.
|
||||
|
||||
Args:
|
||||
path: Path to the file
|
||||
change_type: Type of change detected
|
||||
file_service: Service to read file and compute checksum
|
||||
|
||||
Returns:
|
||||
FileChange with computed checksum for non-deleted files
|
||||
"""
|
||||
file_path = file_service.path(path)
|
||||
content, checksum = await file_service.read_file(file_path) if change_type != Change.deleted else (None, None)
|
||||
return cls(path=file_path, change_type=change_type, checksum=checksum)
|
||||
from typing import Set, Dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncReport:
|
||||
"""Report of file changes found compared to database state.
|
||||
|
||||
|
||||
Attributes:
|
||||
total: Total number of files in directory being synced
|
||||
new: Files that exist on disk but not in database
|
||||
@@ -48,19 +16,16 @@ class SyncReport:
|
||||
moves: Files that have been moved from one location to another
|
||||
checksums: Current checksums for files on disk
|
||||
"""
|
||||
|
||||
total: int = 0
|
||||
# We keep paths as strings in sets/dicts for easier serialization
|
||||
new: Set[str] = field(default_factory=set)
|
||||
modified: Set[str] = field(default_factory=set)
|
||||
deleted: Set[str] = field(default_factory=set)
|
||||
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
|
||||
checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
|
||||
moves: Dict[str, str] = field(default_factory=dict) # old_path -> new_path
|
||||
checksums: Dict[str, str] = field(default_factory=dict) # path -> checksum
|
||||
|
||||
@property
|
||||
def total_changes(self) -> int:
|
||||
"""Total number of changes."""
|
||||
return len(self.new) + len(self.modified) + len(self.deleted) + len(self.moves)
|
||||
|
||||
@property
|
||||
def syned_files(self) -> int:
|
||||
"""Total number of files synced."""
|
||||
return len(self.new) + len(self.modified) + len(self.moves)
|
||||
|
||||
@@ -8,7 +8,6 @@ from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
from rich import box
|
||||
from rich.console import Console
|
||||
from rich.live import Live
|
||||
from rich.table import Table
|
||||
@@ -84,7 +83,7 @@ class WatchService:
|
||||
|
||||
def generate_table(self) -> Table:
|
||||
"""Generate status display table"""
|
||||
table = Table(title="Basic Memory Sync Status")
|
||||
table = Table()
|
||||
|
||||
# Add status row
|
||||
table.add_column("Status", style="cyan")
|
||||
@@ -131,13 +130,36 @@ class WatchService:
|
||||
|
||||
return table
|
||||
|
||||
async def run(self):
|
||||
async def run(self, console_status: bool = False): # pragma: no cover
|
||||
"""Watch for file changes and sync them"""
|
||||
logger.info("Watching for sync changes")
|
||||
self.state.running = True
|
||||
self.state.start_time = datetime.now()
|
||||
await self.write_status()
|
||||
|
||||
with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
|
||||
if console_status:
|
||||
with Live(self.generate_table(), refresh_per_second=4, console=self.console) as live:
|
||||
try:
|
||||
async for changes in awatch(
|
||||
self.config.home,
|
||||
watch_filter=self.filter_changes,
|
||||
debounce=self.config.sync_delay,
|
||||
recursive=True,
|
||||
):
|
||||
# Process changes
|
||||
await self.handle_changes(self.config.home)
|
||||
# Update display
|
||||
live.update(self.generate_table())
|
||||
|
||||
except Exception as e:
|
||||
self.state.record_error(str(e))
|
||||
await self.write_status()
|
||||
raise
|
||||
finally:
|
||||
self.state.running = False
|
||||
await self.write_status()
|
||||
|
||||
else:
|
||||
try:
|
||||
async for changes in awatch(
|
||||
self.config.home,
|
||||
@@ -148,7 +170,6 @@ class WatchService:
|
||||
# Process changes
|
||||
await self.handle_changes(self.config.home)
|
||||
# Update display
|
||||
live.update(self.generate_table())
|
||||
|
||||
except Exception as e:
|
||||
self.state.record_error(str(e))
|
||||
|
||||
+42
-33
@@ -1,38 +1,18 @@
|
||||
"""Utility functions for basic-memory."""
|
||||
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from loguru import logger
|
||||
from unidecode import unidecode
|
||||
|
||||
|
||||
def sanitize_name(name: str) -> str:
|
||||
"""
|
||||
Sanitize a name for filesystem use:
|
||||
- Convert to lowercase
|
||||
- Replace spaces/punctuation with underscores
|
||||
- Remove emojis and other special characters
|
||||
- Collapse multiple underscores
|
||||
- Trim leading/trailing underscores
|
||||
"""
|
||||
# Normalize unicode to compose characters where possible
|
||||
name = unicodedata.normalize("NFKD", name)
|
||||
# Remove emojis and other special characters, keep only letters, numbers, spaces
|
||||
name = "".join(c for c in name if c.isalnum() or c.isspace())
|
||||
# Replace spaces with underscores
|
||||
name = name.replace(" ", "_")
|
||||
# Remove newline
|
||||
name = name.replace("\n", "")
|
||||
# Convert to lowercase
|
||||
name = name.lower()
|
||||
# Collapse multiple underscores and trim
|
||||
name = re.sub(r"_+", "_", name).strip("_")
|
||||
|
||||
return name
|
||||
from basic_memory.config import config
|
||||
|
||||
|
||||
def generate_permalink(file_path: Path | str) -> str:
|
||||
def generate_permalink(file_path: Union[Path, str]) -> str:
|
||||
"""Generate a stable permalink from a file path.
|
||||
|
||||
Args:
|
||||
@@ -50,8 +30,11 @@ def generate_permalink(file_path: Path | str) -> str:
|
||||
>>> generate_permalink("design/unified_model_refactor.md")
|
||||
'design/unified-model-refactor'
|
||||
"""
|
||||
# Convert Path to string if needed
|
||||
path_str = str(file_path)
|
||||
|
||||
# Remove extension
|
||||
base = os.path.splitext(file_path)[0]
|
||||
base = os.path.splitext(path_str)[0]
|
||||
|
||||
# Transliterate unicode to ascii
|
||||
ascii_text = unidecode(base)
|
||||
@@ -63,16 +46,42 @@ def generate_permalink(file_path: Path | str) -> str:
|
||||
lower_text = ascii_text.lower()
|
||||
|
||||
# replace underscores with hyphens
|
||||
text_with_hyphens = lower_text.replace('_', '-')
|
||||
text_with_hyphens = lower_text.replace("_", "-")
|
||||
|
||||
# Replace remaining invalid chars with hyphens
|
||||
clean_text = re.sub(r'[^a-z0-9/\-]', '-', text_with_hyphens)
|
||||
clean_text = re.sub(r"[^a-z0-9/\-]", "-", text_with_hyphens)
|
||||
|
||||
# Collapse multiple hyphens
|
||||
clean_text = re.sub(r'-+', '-', clean_text)
|
||||
clean_text = re.sub(r"-+", "-", clean_text)
|
||||
|
||||
# Clean each path segment
|
||||
segments = clean_text.split('/')
|
||||
clean_segments = [s.strip('-') for s in segments]
|
||||
segments = clean_text.split("/")
|
||||
clean_segments = [s.strip("-") for s in segments]
|
||||
|
||||
return '/'.join(clean_segments)
|
||||
return "/".join(clean_segments)
|
||||
|
||||
|
||||
def setup_logging(home_dir: Path = config.home, log_file: Optional[str] = None) -> None:
|
||||
"""
|
||||
Configure logging for the application.
|
||||
"""
|
||||
|
||||
# Remove default handler and any existing handlers
|
||||
logger.remove()
|
||||
|
||||
# Add file handler
|
||||
if log_file:
|
||||
log_path = home_dir / log_file
|
||||
logger.add(
|
||||
str(log_path), # loguru expects a string path
|
||||
level=config.log_level,
|
||||
rotation="100 MB",
|
||||
retention="10 days",
|
||||
backtrace=True,
|
||||
diagnose=True,
|
||||
enqueue=True,
|
||||
colorize=False,
|
||||
)
|
||||
|
||||
# Add stderr handler
|
||||
logger.add(sys.stderr, level=config.log_level, backtrace=True, diagnose=True, colorize=True)
|
||||
|
||||
@@ -1,458 +0,0 @@
|
||||
# Basic Memory Tasks
|
||||
|
||||
## Current Focus
|
||||
|
||||
### Observation Management
|
||||
|
||||
Implement update/remove functionality for observations with a focus on maintainability and consistency with our "
|
||||
filesystem is source of truth" principle.
|
||||
|
||||
Options under consideration:
|
||||
|
||||
1. Bulk Update Approach
|
||||
- Update all observations at once
|
||||
- Pros:
|
||||
- Simpler file operations
|
||||
- No need to match on observation content
|
||||
- Easier database synchronization
|
||||
- Very consistent with "filesystem is source of truth"
|
||||
- Cons:
|
||||
- Less efficient - rewrites everything for small changes
|
||||
- Potential concurrency implications
|
||||
|
||||
2. Tracked Observations Approach
|
||||
- Use markdown comments for observation IDs
|
||||
```markdown
|
||||
# Entity Name
|
||||
type: entity_type
|
||||
|
||||
## Observations
|
||||
- <!-- obs-id: abc123 -->
|
||||
This is an observation
|
||||
```
|
||||
- Pros:
|
||||
- Can track individual observations
|
||||
- Enables precise updates/deletes
|
||||
- Cons:
|
||||
- More complex markdown parsing
|
||||
- IDs visible in markdown
|
||||
|
||||
3. Diff-based Approach
|
||||
- Implement observation-aware diffing
|
||||
- Track changes at observation level
|
||||
- Pros:
|
||||
- More efficient updates
|
||||
- Preserves manual edits
|
||||
- Cons:
|
||||
- More complex implementation
|
||||
- Need to handle merge conflicts
|
||||
|
||||
4. Position-based Management
|
||||
- Track observations by their position/order
|
||||
- Pros:
|
||||
- No need for explicit IDs
|
||||
- Clean markdown
|
||||
- Cons:
|
||||
- Fragile if order changes
|
||||
- Hard to handle concurrent edits
|
||||
|
||||
## Completed
|
||||
|
||||
- [x] Extract file operations to fileio.py module
|
||||
- [x] Update EntityService to use fileio functions
|
||||
- [x] Initial ObservationService implementation
|
||||
- [x] Basic test coverage
|
||||
|
||||
## Future Work
|
||||
|
||||
- [ ] Implement observation updates/removals (exploring options above)
|
||||
- [ ] Proper session management for concurrent operations
|
||||
- [ ] EntityService tests using new fileio module
|
||||
- [ ] More sophisticated search functionality
|
||||
- [ ] Handle markdown formatting edge cases
|
||||
|
||||
## TODO
|
||||
|
||||
### refactor input schema
|
||||
|
||||
1. Observations Format:
|
||||
Old (JSON) way I tried first:
|
||||
|
||||
```python
|
||||
"observations": ["First observation", "Second observation"] # Simple string array
|
||||
```
|
||||
|
||||
New required format:
|
||||
|
||||
```python
|
||||
"observations": [
|
||||
{"content": "First observation"},
|
||||
{"content": "Second observation"}
|
||||
] # Array of objects with content field
|
||||
```
|
||||
|
||||
2. Relations Format:
|
||||
Old way:
|
||||
|
||||
```python
|
||||
"relations": [
|
||||
{"from": "EntityName", "to": "OtherEntity", "relationType": "relates_to"} # Using names
|
||||
]
|
||||
```
|
||||
|
||||
New format:
|
||||
|
||||
```python
|
||||
"relations": [
|
||||
{"fromId": "20241210-entity-id", "toId": "20241210-other-id", "relationType": "relates_to"} # Using IDs
|
||||
]
|
||||
```
|
||||
|
||||
My preferences:
|
||||
|
||||
1. For observations: The simple string array felt more intuitive for basic use, but I can see the benefits of the object
|
||||
format:
|
||||
- Allows for additional metadata (context, timestamps, etc.)
|
||||
- More explicit about what each field means
|
||||
- Consistent with how we'd want to store this in a database
|
||||
|
||||
2. For relations: Using IDs is technically better but requires an extra lookup step in my workflow:
|
||||
- I have to first create the entities to get their IDs
|
||||
- Then use those IDs to create relations
|
||||
- Makes it harder to create entities and relations in a single step
|
||||
|
||||
Suggestions for making it more intuitive:
|
||||
|
||||
1. For observations: We could have a helper function that accepts either format:
|
||||
|
||||
```python
|
||||
# Both would work:
|
||||
create_entities([{
|
||||
"name": "Entity",
|
||||
"entityType": "type",
|
||||
"observations": ["Simple string obs"] # Auto-converted to object format
|
||||
}])
|
||||
|
||||
create_entities([{
|
||||
"name": "Entity",
|
||||
"entityType": "type",
|
||||
"observations": [{"content": "Full object obs"}] # Native format
|
||||
}])
|
||||
```
|
||||
|
||||
2. For relations: Maybe allow a name-based helper function:
|
||||
|
||||
```python
|
||||
# Instead of requiring IDs:
|
||||
create_relations_by_name([{
|
||||
"from": "EntityName",
|
||||
"to": "OtherEntity",
|
||||
"relationType": "relates_to"
|
||||
}])
|
||||
```
|
||||
|
||||
3. A combined creation function for when we want to create entities and their relations together:
|
||||
|
||||
```python
|
||||
create_entity_with_relations({
|
||||
"entity": {
|
||||
"name": "NewEntity",
|
||||
"entityType": "type",
|
||||
"observations": ["Obs 1", "Obs 2"]
|
||||
},
|
||||
"relations": [{
|
||||
"to": "ExistingEntity",
|
||||
"relationType": "relates_to"
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
The current schema is more "correct" from a software engineering perspective, but these helpers could make it more
|
||||
natural to use while maintaining the rigorous underlying structure. What do you think about these suggestions? Would you
|
||||
prefer to keep it strict for clarity, or add some convenience layers?
|
||||
|
||||
### entity file organization
|
||||
|
||||
- my problems
|
||||
- wall of text is hard to scan visually
|
||||
- files are not ordered or grouped granular enough. Only `<date>_<entity_name>.md`
|
||||
- hard to tell when a new entity file is added
|
||||
|
||||
Possible fixes:
|
||||
|
||||
- use subdirectories?
|
||||
|
||||
## TASKS
|
||||
|
||||
1. **Core Functionality Improvements**
|
||||
- [x] entity.description addition
|
||||
- [x] subdirectories
|
||||
- Paul needs this for markdown view
|
||||
- [x] refine directory sprawl
|
||||
- [ ] improve tool api
|
||||
- [ ] Choose and implement observation update/removal strategy
|
||||
- [ ] Handle relationship updates in files
|
||||
- [ ] Complete full CRUD operations
|
||||
- delete
|
||||
- [ ] Improve search functionality (currently broken as we discovered)
|
||||
|
||||
### Suggested Sequence
|
||||
|
||||
1. **Schema Update First**
|
||||
- Add `entity.description` field
|
||||
- rename entity.references?
|
||||
- This affects database, Pydantic models, and file format
|
||||
- Good foundation for other changes
|
||||
|
||||
2. **File Organization**
|
||||
- Add subdirectory support
|
||||
- Affects:
|
||||
- File path handling
|
||||
- Entity loading/saving
|
||||
- URI resolution
|
||||
- Will make Paul's markdown viewing experience better
|
||||
|
||||
3. **Tool API Improvements**
|
||||
- Cleaner input/output schemas
|
||||
- More consistent patterns
|
||||
- Better error handling
|
||||
- This sets us up for implementing the remaining operations
|
||||
|
||||
4. **Core Operations**
|
||||
- Implement delete operations
|
||||
- Update/remove observations
|
||||
- Relationship updates in files
|
||||
- Building on the improved API
|
||||
|
||||
5. **Search Fix**
|
||||
- Can properly tackle this after file organization
|
||||
- Will benefit from improved schema
|
||||
|
||||
Would you like me to:
|
||||
1. Start with the schema update for entity.description?
|
||||
2. Plan out the subdirectory implementation?
|
||||
3. Or focus on a different area?
|
||||
|
||||
I think the schema update would be a clean, contained change to start with, but I'm happy to tackle whichever part you think would be most valuable first.
|
||||
|
||||
|
||||
2. **Robustness & Testing**
|
||||
- Fix DI issues
|
||||
- Learn from fastmcp patterns
|
||||
- Markdown service
|
||||
- markdown.py
|
||||
-python-frontmatter
|
||||
- Complete test coverage
|
||||
- Expand testing across services
|
||||
- 100% coverate
|
||||
- Improve error handling and logging
|
||||
- Add comprehensive type hints
|
||||
|
||||
3. **Architecture Improvements**
|
||||
- Handle concurrent file operations safely
|
||||
- Implement proper session management
|
||||
- Balance file operations and DB sync
|
||||
- Handle markdown formatting edge cases
|
||||
|
||||
4. **Documentation & Infrastructure**
|
||||
- Document core modules
|
||||
- Implement proper backup strategy
|
||||
- Add file change versioning
|
||||
- Improve CLI interface
|
||||
|
||||
|
||||
## Ideas
|
||||
|
||||
- need update tool
|
||||
|
||||
### 2-way sync
|
||||
|
||||
- Enable updates to the markdown files to be able to be seen by AI
|
||||
- possible via tool sync
|
||||
- filesystem notifications via agent?
|
||||
- Claude can use `file_write` tool to edit Entity files also
|
||||
|
||||
### Projects
|
||||
|
||||
- support multiple projects
|
||||
- figure out flow
|
||||
- load project at startup?
|
||||
- switch project during chat?
|
||||
|
||||
### References
|
||||
|
||||
- better support for ref:// references
|
||||
- should we call them memory://<project>/<entity>
|
||||
- use Prompt to invoke reference uri
|
||||
|
||||
### Entity File format
|
||||
|
||||
- Consider using wiki link format:
|
||||
[[20241210-ai_human_collaboration_model]] improves
|
||||
|
||||
instead of
|
||||
[20241210-basic-memory] validates_design_of
|
||||
|
||||
- use frontmatter for file header
|
||||
|
||||
### Code
|
||||
|
||||
- cleanup server.py
|
||||
- use markdown.py?
|
||||
- fastmcp?
|
||||
- Entity.description
|
||||
|
||||
### Other Features
|
||||
|
||||
- Search
|
||||
- webui
|
||||
- python notebook?
|
||||
- vector eb extension for sqlite
|
||||
|
||||
#### Obsidian
|
||||
|
||||
- use as file viewer editor
|
||||
- create knowledge graph?
|
||||
|
||||
#### Save chats
|
||||
|
||||
I'll sketch out a potential schema for chat sessions in our system:
|
||||
|
||||
```
|
||||
{
|
||||
`entities`: [
|
||||
{
|
||||
`name`: `Chat_Entity_Design`,
|
||||
`entityType`: `design_proposal`,
|
||||
`observations`: [
|
||||
{
|
||||
`content`: `Entity Schema:
|
||||
type: chat_session
|
||||
date: ISO timestamp
|
||||
participants: [human, ai]
|
||||
topic: string
|
||||
context_refs: [memory://<project>/<entity_id>]
|
||||
key_decisions: [string]
|
||||
summary: string`
|
||||
},
|
||||
{
|
||||
`content`: `File Format (frontmatter):
|
||||
---
|
||||
type: chat_session
|
||||
date: 2024-12-10T10:30:00Z
|
||||
participants:
|
||||
- human: paul
|
||||
- ai: claude
|
||||
topic: Chat Session Storage Design
|
||||
context_refs:
|
||||
- memory://basic-memory/20241210-chat-entity-design
|
||||
key_decisions:
|
||||
- Store chats as first-class entities
|
||||
- Use memory:// URI scheme
|
||||
summary: Designed schema for chat storage
|
||||
---`
|
||||
},
|
||||
{
|
||||
`content`: `Example Relations:
|
||||
- chat_session references design_proposal
|
||||
- chat_session implements feature
|
||||
- chat_session discusses concept
|
||||
- chat_session continues_from previous_chat`
|
||||
},
|
||||
{
|
||||
`content`: `Auto-generation approach:
|
||||
1. At end of chat, Claude summarizes key points
|
||||
2. Creates chat entity with metadata
|
||||
3. Adds relations to referenced entities
|
||||
4. Updates any entities modified during chat`
|
||||
},
|
||||
{
|
||||
`content`: `Integration points:
|
||||
- memory:// URI scheme for references
|
||||
- Relations to track context chain
|
||||
- File format matches entity schema
|
||||
- Automatic metadata generation`
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
{
|
||||
`entities`: [
|
||||
{
|
||||
`name`: `Chat_Session_Workflow`,
|
||||
`entityType`: `process`,
|
||||
`observations`: [
|
||||
{
|
||||
`content`: `Start of chat:
|
||||
- Load any referenced context via memory:// URIs
|
||||
- Initialize new chat session metadata
|
||||
- Track entities being discussed/modified`
|
||||
},
|
||||
{
|
||||
`content`: `During chat:
|
||||
- Update references as new entities are mentioned
|
||||
- Track key decisions and changes
|
||||
- Note any new relations discovered`
|
||||
},
|
||||
{
|
||||
`content`: `End of chat:
|
||||
- Generate summary and key points
|
||||
- Create chat session entity
|
||||
- Add relations to referenced entities
|
||||
- Update any modified entities`
|
||||
},
|
||||
{
|
||||
`content`: `Future chat reference:
|
||||
- Can reference previous chat via memory:// URI
|
||||
- Loads relevant context and decisions
|
||||
- Creates continues_from relation automatically`
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Key aspects of this design:
|
||||
|
||||
1. Chat Session as Entity
|
||||
|
||||
- Clear metadata (participants, date, topic)
|
||||
- Explicit references to other entities via memory:// URIs
|
||||
- Captures key decisions and summary
|
||||
- Uses frontmatter for clean file format
|
||||
|
||||
2. Rich Relations
|
||||
|
||||
- Can track chat-to-chat continuity
|
||||
- Links to entities discussed/modified
|
||||
- Captures purpose (discusses, implements, etc.)
|
||||
- Maintains context chain
|
||||
|
||||
3. Automated Workflow
|
||||
|
||||
- I could generate metadata during chat
|
||||
- Create summary and relations at end
|
||||
- Update any modified entities
|
||||
- Makes context loading explicit
|
||||
|
||||
4. Integration
|
||||
|
||||
- memory:// URI scheme fits naturally
|
||||
- File format matches our entity pattern
|
||||
- Two-way links between chats and entities
|
||||
- Clear context persistence model
|
||||
|
||||
The nice thing about this design is that it makes chat sessions "first-class citizens" in our knowledge graph while
|
||||
keeping everything in human-readable markdown files.
|
||||
|
||||
Would you like me to:
|
||||
|
||||
1. Sketch out what this chat would look like as an entity?
|
||||
2. Detail the MCP tool changes needed?
|
||||
3. Mock up the file format more precisely?
|
||||
4. Something else?
|
||||
@@ -77,8 +77,6 @@ async def test_create_entity_observations_relations(client: AsyncClient, file_se
|
||||
assert entity.relations[0].from_id == "test/test-entity"
|
||||
assert entity.relations[0].to_id is None
|
||||
|
||||
# TODO Relation.to_id should be name from link
|
||||
|
||||
# Verify file has new content but preserved metadata
|
||||
file_path = file_service.get_entity_path(entity)
|
||||
file_content, _ = await file_service.read_file(file_path)
|
||||
@@ -111,8 +109,12 @@ async def test_get_entity(client: AsyncClient):
|
||||
async def test_get_entities(client: AsyncClient):
|
||||
"""Should open multiple entities by path IDs."""
|
||||
# Create a few entities with different names
|
||||
await client.post("/knowledge/entities", json={"title": "AlphaTest", "folder": "", "entity_type": "test"})
|
||||
await client.post("/knowledge/entities", json={"title": "BetaTest", "folder": "", "entity_type": "test"})
|
||||
await client.post(
|
||||
"/knowledge/entities", json={"title": "AlphaTest", "folder": "", "entity_type": "test"}
|
||||
)
|
||||
await client.post(
|
||||
"/knowledge/entities", json={"title": "BetaTest", "folder": "", "entity_type": "test"}
|
||||
)
|
||||
|
||||
# Open nodes by path IDs
|
||||
response = await client.get(
|
||||
@@ -397,7 +399,12 @@ async def test_update_entity_type_conversion(client: AsyncClient):
|
||||
async def test_update_entity_metadata(client: AsyncClient):
|
||||
"""Test updating entity metadata."""
|
||||
# Create entity
|
||||
data = {"title": "test", "folder": "", "entity_type": "test", "entity_metadata": {"status": "draft"}}
|
||||
data = {
|
||||
"title": "test",
|
||||
"folder": "",
|
||||
"entity_type": "test",
|
||||
"entity_metadata": {"status": "draft"},
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
|
||||
@@ -450,7 +457,12 @@ async def test_update_entity_incorrect_permalink(client: AsyncClient):
|
||||
async def test_update_entity_search_index(client: AsyncClient):
|
||||
"""Test search index is updated after entity changes."""
|
||||
# Create entity
|
||||
data = {"title": "test", "folder": "", "entity_type": "test", "content": "Initial searchable content"}
|
||||
data = {
|
||||
"title": "test",
|
||||
"folder": "",
|
||||
"entity_type": "test",
|
||||
"content": "Initial searchable content",
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ async def test_get_memory_context(client, test_graph):
|
||||
assert context.metadata.depth == 1 # default depth
|
||||
# assert context.metadata["timeframe"] == "7d" # default timeframe
|
||||
assert isinstance(context.metadata.generated_at, datetime)
|
||||
assert context.metadata.total_results == 2
|
||||
assert context.metadata.total_results == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -69,15 +69,6 @@ async def test_get_memory_context_timeframe(client, test_graph):
|
||||
assert len(older.related_results) >= len(recent.related_results)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_related_context_filters(client, test_graph):
|
||||
"""Test filtering related content by relation type."""
|
||||
response = await client.get("/memory/related/test/root")
|
||||
assert response.status_code == 200
|
||||
|
||||
context = GraphContext(**response.json())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_found(client):
|
||||
"""Test handling of non-existent paths."""
|
||||
|
||||
@@ -35,6 +35,7 @@ async def test_get_resource_content(client, test_config, entity_repository):
|
||||
assert response.text == content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_by_title(client, test_config, entity_repository):
|
||||
"""Test getting content by permalink."""
|
||||
# Create a test file
|
||||
|
||||
@@ -11,7 +11,6 @@ from basic_memory.schemas import Entity as EntitySchema
|
||||
from basic_memory.schemas.search import SearchItemType, SearchResponse
|
||||
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def indexed_entity(init_search_index, full_entity, search_service):
|
||||
"""Create an entity and index it."""
|
||||
@@ -22,17 +21,17 @@ async def indexed_entity(init_search_index, full_entity, search_service):
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_basic(client, indexed_entity):
|
||||
"""Test basic text search."""
|
||||
response = await client.post("/search/", json={"text": "searchable"})
|
||||
response = await client.post("/search/", json={"text": "search"})
|
||||
assert response.status_code == 200
|
||||
search_results = SearchResponse.model_validate(response.json())
|
||||
assert len(search_results.results) == 3
|
||||
|
||||
|
||||
found = False
|
||||
for r in search_results.results:
|
||||
if r.type == SearchItemType.ENTITY.value:
|
||||
if r.type == SearchItemType.ENTITY.value:
|
||||
assert r.permalink == indexed_entity.permalink
|
||||
found = True
|
||||
|
||||
|
||||
assert found, "Expected to find indexed entity in results"
|
||||
|
||||
|
||||
@@ -45,13 +44,15 @@ async def test_search_with_type_filter(client, indexed_entity):
|
||||
)
|
||||
assert response.status_code == 200
|
||||
search_results = SearchResponse.model_validate(response.json())
|
||||
assert len(search_results.results) > 0
|
||||
|
||||
# Should not find with wrong type
|
||||
# Should find with relation type
|
||||
response = await client.post(
|
||||
"/search/", json={"text": "test", "types": [SearchItemType.RELATION.value]}
|
||||
)
|
||||
assert response.status_code == 200
|
||||
search_results = SearchResponse.model_validate(response.json())
|
||||
assert len(search_results.results) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -90,6 +91,7 @@ async def test_search_with_date_filter(client, indexed_entity):
|
||||
search_results = SearchResponse.model_validate(response.json())
|
||||
assert len(search_results.results) == 0
|
||||
|
||||
|
||||
@pytest.mark.skip("search scoring is not implemented yet")
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_scoring(client, indexed_entity):
|
||||
@@ -152,8 +154,6 @@ async def test_reindex(client, search_service, entity_service, session_maker):
|
||||
search_response = await client.post("/search/", json={"text": "test"})
|
||||
search_results = SearchResponse.model_validate(search_response.json())
|
||||
assert len(search_results.results) == 1
|
||||
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Tests for import_chatgpt command."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import import_app, app
|
||||
from basic_memory.cli.commands import import_chatgpt
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation():
|
||||
"""Sample ChatGPT conversation data for testing."""
|
||||
return {
|
||||
"title": "Test Conversation",
|
||||
"create_time": 1736616594.24054, # Example timestamp
|
||||
"update_time": 1736616603.164995,
|
||||
"mapping": {
|
||||
"root": {"id": "root", "message": None, "parent": None, "children": ["msg1"]},
|
||||
"msg1": {
|
||||
"id": "msg1",
|
||||
"message": {
|
||||
"id": "msg1",
|
||||
"author": {"role": "user", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {"content_type": "text", "parts": ["Hello, this is a test message"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": ["msg2"],
|
||||
},
|
||||
"msg2": {
|
||||
"id": "msg2",
|
||||
"message": {
|
||||
"id": "msg2",
|
||||
"author": {"role": "assistant", "name": None, "metadata": {}},
|
||||
"create_time": 1736616603.164995,
|
||||
"content": {"content_type": "text", "parts": ["This is a test response"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "msg1",
|
||||
"children": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation_with_code():
|
||||
"""Sample conversation with code block."""
|
||||
conversation = {
|
||||
"title": "Code Test",
|
||||
"create_time": 1736616594.24054,
|
||||
"update_time": 1736616603.164995,
|
||||
"mapping": {
|
||||
"root": {"id": "root", "message": None, "parent": None, "children": ["msg1"]},
|
||||
"msg1": {
|
||||
"id": "msg1",
|
||||
"message": {
|
||||
"id": "msg1",
|
||||
"author": {"role": "assistant", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {
|
||||
"content_type": "code",
|
||||
"language": "python",
|
||||
"text": "def hello():\n print('Hello world!')",
|
||||
},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
"msg2": {
|
||||
"id": "msg2",
|
||||
"message": {
|
||||
"id": "msg2",
|
||||
"author": {"role": "assistant", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
return conversation
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation_with_hidden():
|
||||
"""Sample conversation with hidden messages."""
|
||||
conversation = {
|
||||
"title": "Hidden Test",
|
||||
"create_time": 1736616594.24054,
|
||||
"update_time": 1736616603.164995,
|
||||
"mapping": {
|
||||
"root": {
|
||||
"id": "root",
|
||||
"message": None,
|
||||
"parent": None,
|
||||
"children": ["visible", "hidden"],
|
||||
},
|
||||
"visible": {
|
||||
"id": "visible",
|
||||
"message": {
|
||||
"id": "visible",
|
||||
"author": {"role": "user", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {"content_type": "text", "parts": ["Visible message"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
"hidden": {
|
||||
"id": "hidden",
|
||||
"message": {
|
||||
"id": "hidden",
|
||||
"author": {"role": "system", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {"content_type": "text", "parts": ["Hidden message"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {"is_visually_hidden_from_conversation": True},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
return conversation
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_chatgpt_json(tmp_path, sample_conversation):
|
||||
"""Create a sample ChatGPT JSON file."""
|
||||
json_file = tmp_path / "conversations.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation], f)
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_chatgpt_json(tmp_path, sample_chatgpt_json):
|
||||
"""Test importing conversations from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_chatgpt.process_chatgpt_json(sample_chatgpt_json, tmp_path, processor)
|
||||
|
||||
assert results["conversations"] == 1
|
||||
assert results["messages"] == 2
|
||||
|
||||
# Check conversation file exists
|
||||
conv_path = tmp_path / "20250111-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
|
||||
# Check content formatting
|
||||
content = conv_path.read_text()
|
||||
assert "# Test Conversation" in content
|
||||
assert "### User" in content
|
||||
assert "Hello, this is a test message" in content
|
||||
assert "### Assistant" in content
|
||||
assert "This is a test response" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_code_blocks(tmp_path, sample_conversation_with_code):
|
||||
"""Test handling of code blocks."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
# Create test file
|
||||
json_file = tmp_path / "code_test.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation_with_code], f)
|
||||
|
||||
await import_chatgpt.process_chatgpt_json(json_file, tmp_path, processor)
|
||||
|
||||
# Check content
|
||||
conv_path = tmp_path / "20250111-code-test.md"
|
||||
content = conv_path.read_text()
|
||||
assert "```python" in content
|
||||
assert "def hello():" in content
|
||||
assert "```" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hidden_messages(tmp_path, sample_conversation_with_hidden):
|
||||
"""Test handling of hidden messages."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
# Create test file
|
||||
json_file = tmp_path / "hidden_test.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation_with_hidden], f)
|
||||
|
||||
results = await import_chatgpt.process_chatgpt_json(json_file, tmp_path, processor)
|
||||
|
||||
# Should only count visible messages
|
||||
assert results["messages"] == 1
|
||||
|
||||
# Check content
|
||||
conv_path = tmp_path / "20250111-hidden-test.md"
|
||||
content = conv_path.read_text()
|
||||
assert "Visible message" in content
|
||||
assert "Hidden message" not in content
|
||||
|
||||
|
||||
def test_import_chatgpt_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import", "chatgpt", "--conversations-json", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_chatgpt_command_success(tmp_path, sample_chatgpt_json, monkeypatch):
|
||||
"""Test successful conversation import via command."""
|
||||
# Set up test environment
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
import_app, ["chatgpt", "--conversations-json", str(sample_chatgpt_json)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Imported 1 conversations" in result.output
|
||||
assert "Containing 2 messages" in result.output
|
||||
|
||||
|
||||
def test_import_chatgpt_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(import_app, ["chatgpt", "--conversations-json", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_chatgpt_with_custom_folder(tmp_path, sample_chatgpt_json, monkeypatch):
|
||||
"""Test import with custom conversations folder."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
conversations_folder = "chats"
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"import",
|
||||
"chatgpt",
|
||||
"--conversations-json",
|
||||
str(sample_chatgpt_json),
|
||||
"--folder",
|
||||
conversations_folder,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check files in custom folder
|
||||
conv_path = tmp_path / conversations_folder / "20250111-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Tests for import_claude command (chat conversations)."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands import import_claude_conversations as import_claude
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation():
|
||||
"""Sample conversation data for testing."""
|
||||
return {
|
||||
"uuid": "test-uuid",
|
||||
"name": "Test Conversation",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"chat_messages": [
|
||||
{
|
||||
"uuid": "msg-1",
|
||||
"text": "Hello, this is a test",
|
||||
"sender": "human",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"content": [{"type": "text", "text": "Hello, this is a test"}],
|
||||
},
|
||||
{
|
||||
"uuid": "msg-2",
|
||||
"text": "Response to test",
|
||||
"sender": "assistant",
|
||||
"created_at": "2025-01-05T20:55:40.123456+00:00",
|
||||
"content": [{"type": "text", "text": "Response to test"}],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversations_json(tmp_path, sample_conversation):
|
||||
"""Create a sample conversations.json file."""
|
||||
json_file = tmp_path / "conversations.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation], f)
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_chat_json(tmp_path, sample_conversations_json):
|
||||
"""Test importing conversations from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_claude.process_conversations_json(
|
||||
sample_conversations_json, tmp_path, processor
|
||||
)
|
||||
|
||||
assert results["conversations"] == 1
|
||||
assert results["messages"] == 2
|
||||
|
||||
# Check conversation file
|
||||
conv_path = tmp_path / "20250105-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
content = conv_path.read_text()
|
||||
|
||||
# Check content formatting
|
||||
assert "### Human" in content
|
||||
assert "Hello, this is a test" in content
|
||||
assert "### Assistant" in content
|
||||
assert "Response to test" in content
|
||||
|
||||
|
||||
def test_import_conversations_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import", "claude", "conversations", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_conversations_command_success(tmp_path, sample_conversations_json, monkeypatch):
|
||||
"""Test successful conversation import via command."""
|
||||
# Set up test environment
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app, ["import", "claude", "conversations", str(sample_conversations_json)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Imported 1 conversations" in result.output
|
||||
assert "Containing 2 messages" in result.output
|
||||
|
||||
|
||||
def test_import_conversations_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(app, ["import", "claude", "conversations", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_conversations_with_custom_folder(tmp_path, sample_conversations_json, monkeypatch):
|
||||
"""Test import with custom conversations folder."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
conversations_folder = "chats"
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"import",
|
||||
"claude",
|
||||
"conversations",
|
||||
str(sample_conversations_json),
|
||||
"--folder",
|
||||
conversations_folder,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check files in custom folder
|
||||
conv_path = tmp_path / conversations_folder / "20250105-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
|
||||
|
||||
def test_import_conversation_with_attachments(tmp_path):
|
||||
"""Test importing conversation with attachments."""
|
||||
# Create conversation with attachments
|
||||
conversation = {
|
||||
"uuid": "test-uuid",
|
||||
"name": "Test With Attachments",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"chat_messages": [
|
||||
{
|
||||
"uuid": "msg-1",
|
||||
"text": "Here's a file",
|
||||
"sender": "human",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"content": [{"type": "text", "text": "Here's a file"}],
|
||||
"attachments": [
|
||||
{"file_name": "test.txt", "extracted_content": "Test file content"}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
json_file = tmp_path / "with_attachments.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([conversation], f)
|
||||
|
||||
# Set up environment
|
||||
config.home = tmp_path
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import", "claude", "conversations", str(json_file)])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check attachment formatting
|
||||
conv_path = tmp_path / "conversations/20250105-test-with-attachments.md"
|
||||
content = conv_path.read_text()
|
||||
assert "**Attachment: test.txt**" in content
|
||||
assert "```" in content
|
||||
assert "Test file content" in content
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Tests for import_claude_projects command."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands import import_claude_projects
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_project():
|
||||
"""Sample project data for testing."""
|
||||
return {
|
||||
"uuid": "test-uuid",
|
||||
"name": "Test Project",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"prompt_template": "# Test Prompt\n\nThis is a test prompt.",
|
||||
"docs": [
|
||||
{
|
||||
"uuid": "doc-uuid-1",
|
||||
"filename": "Test Document",
|
||||
"content": "# Test Document\n\nThis is test content.",
|
||||
"created_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
},
|
||||
{
|
||||
"uuid": "doc-uuid-2",
|
||||
"filename": "Another Document",
|
||||
"content": "# Another Document\n\nMore test content.",
|
||||
"created_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_projects_json(tmp_path, sample_project):
|
||||
"""Create a sample projects.json file."""
|
||||
json_file = tmp_path / "projects.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_project], f)
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_projects_json(tmp_path, sample_projects_json):
|
||||
"""Test importing projects from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_claude_projects.process_projects_json(
|
||||
sample_projects_json, tmp_path, processor
|
||||
)
|
||||
|
||||
assert results["documents"] == 2
|
||||
assert results["prompts"] == 1
|
||||
|
||||
# Check project directory structure
|
||||
project_dir = tmp_path / "test-project"
|
||||
assert project_dir.exists()
|
||||
assert (project_dir / "docs").exists()
|
||||
assert (project_dir / "prompt-template.md").exists()
|
||||
|
||||
# Check document files
|
||||
doc1 = project_dir / "docs/test-document.md"
|
||||
assert doc1.exists()
|
||||
content1 = doc1.read_text()
|
||||
assert "# Test Document" in content1
|
||||
assert "This is test content" in content1
|
||||
|
||||
# Check prompt template
|
||||
prompt = project_dir / "prompt-template.md"
|
||||
assert prompt.exists()
|
||||
prompt_content = prompt.read_text()
|
||||
assert "# Test Prompt" in prompt_content
|
||||
assert "This is a test prompt" in prompt_content
|
||||
|
||||
|
||||
def test_import_projects_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_projects_command_success(tmp_path, sample_projects_json, monkeypatch):
|
||||
"""Test successful project import via command."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(sample_projects_json)])
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Imported 2 project documents" in result.output
|
||||
assert "Imported 1 prompt templates" in result.output
|
||||
|
||||
|
||||
def test_import_projects_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_projects_with_base_folder(tmp_path, sample_projects_json, monkeypatch):
|
||||
"""Test import with custom base folder."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
base_folder = "claude-exports"
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"import",
|
||||
"claude",
|
||||
"projects",
|
||||
str(sample_projects_json),
|
||||
"--base-folder",
|
||||
base_folder,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check files in base folder
|
||||
project_dir = tmp_path / base_folder / "test-project"
|
||||
assert project_dir.exists()
|
||||
assert (project_dir / "docs").exists()
|
||||
assert (project_dir / "prompt-template.md").exists()
|
||||
|
||||
|
||||
def test_import_project_without_prompt(tmp_path):
|
||||
"""Test importing project without prompt template."""
|
||||
# Create project without prompt
|
||||
project = {
|
||||
"uuid": "test-uuid",
|
||||
"name": "No Prompt Project",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"docs": [
|
||||
{
|
||||
"uuid": "doc-uuid-1",
|
||||
"filename": "Test Document",
|
||||
"content": "# Test Document\n\nContent.",
|
||||
"created_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
json_file = tmp_path / "no_prompt.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([project], f)
|
||||
|
||||
# Set up environment
|
||||
config.home = tmp_path
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(json_file)])
|
||||
assert result.exit_code == 0
|
||||
assert "Imported 1 project documents" in result.output
|
||||
assert "Imported 0 prompt templates" in result.output
|
||||
@@ -1,175 +1,134 @@
|
||||
"""Test import-json command functionality."""
|
||||
"""Tests for import_memory_json command."""
|
||||
|
||||
import json
|
||||
from io import StringIO
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from rich.console import Console
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.commands.import_memory_json import process_memory_json
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.cli.commands import import_memory_json
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def console():
|
||||
"""Create test console that captures output."""
|
||||
output = StringIO()
|
||||
return Console(file=output), output
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_memory_json(tmp_path) -> Path:
|
||||
"""Create a sample memory.json file with test data."""
|
||||
json_path = tmp_path / "memory.json"
|
||||
|
||||
# Create test data modeling the real format
|
||||
test_data = [
|
||||
def sample_entities():
|
||||
"""Sample entities for testing."""
|
||||
return [
|
||||
{
|
||||
"type": "entity",
|
||||
"name": "Basic_Memory",
|
||||
"entityType": "software_system",
|
||||
"observations": [
|
||||
"A core component of Basic Machines",
|
||||
"Local-first knowledge management system",
|
||||
"Combines filesystem persistence with graph-based knowledge representation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "entity",
|
||||
"name": "Basic_Machines",
|
||||
"entityType": "project",
|
||||
"observations": [
|
||||
"Local-first knowledge management system",
|
||||
"Focuses on enhancing human agency and understanding",
|
||||
"Current focus includes basic-memory system"
|
||||
]
|
||||
"name": "test_entity",
|
||||
"entityType": "test",
|
||||
"observations": ["Test observation 1", "Test observation 2"],
|
||||
},
|
||||
{
|
||||
"type": "relation",
|
||||
"from": "Basic_Memory",
|
||||
"to": "Basic_Machines",
|
||||
"relationType": "is_component_of"
|
||||
}
|
||||
"from": "test_entity",
|
||||
"to": "related_entity",
|
||||
"relationType": "test_relation",
|
||||
},
|
||||
]
|
||||
|
||||
# Write each item as a JSON line
|
||||
with open(json_path, 'w') as f:
|
||||
for item in test_data:
|
||||
f.write(json.dumps(item) + '\n')
|
||||
|
||||
return json_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_json_file(tmp_path, sample_entities):
|
||||
"""Create a sample memory.json file."""
|
||||
json_file = tmp_path / "memory.json"
|
||||
with open(json_file, "w") as f:
|
||||
for entity in sample_entities:
|
||||
f.write(json.dumps(entity) + "\n")
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_memory_json(
|
||||
sample_memory_json: Path,
|
||||
markdown_processor: MarkdownProcessor,
|
||||
test_config,
|
||||
):
|
||||
"""Test importing from memory.json format."""
|
||||
# Process the import
|
||||
results = await process_memory_json(sample_memory_json, test_config.home, markdown_processor)
|
||||
|
||||
# Check results
|
||||
assert results["entities"] == 2
|
||||
async def test_process_memory_json(tmp_path, sample_json_file):
|
||||
"""Test importing entities from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_memory_json.process_memory_json(sample_json_file, tmp_path, processor)
|
||||
|
||||
assert results["entities"] == 1
|
||||
assert results["relations"] == 1
|
||||
|
||||
# Verify Basic_Memory entity file was created correctly
|
||||
basic_memory_path = test_config.home / "software_system/Basic_Memory.md"
|
||||
entity = await markdown_processor.read_file(basic_memory_path)
|
||||
|
||||
assert entity.frontmatter.title == "Basic_Memory"
|
||||
assert entity.frontmatter.type == "software_system"
|
||||
assert len(entity.observations) == 3
|
||||
assert len(entity.relations) == 1 # Should have the outgoing relation
|
||||
assert entity.relations[0].type == "is_component_of"
|
||||
assert entity.relations[0].target == "Basic_Machines"
|
||||
|
||||
# Verify Basic_Machines entity file
|
||||
basic_machines_path = test_config.home / "project/Basic_Machines.md"
|
||||
entity = await markdown_processor.read_file(basic_machines_path)
|
||||
|
||||
assert entity.frontmatter.title == "Basic_Machines"
|
||||
assert entity.frontmatter.type == "project"
|
||||
assert len(entity.observations) == 3
|
||||
assert len(entity.relations) == 0 # No outgoing relations
|
||||
|
||||
# Check file was created
|
||||
entity_file = tmp_path / "test/test_entity.md"
|
||||
assert entity_file.exists()
|
||||
content = entity_file.read_text()
|
||||
assert "Test observation 1" in content
|
||||
assert "Test observation 2" in content
|
||||
assert "test_relation [[related_entity]]" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_memory_json_empty_observations(
|
||||
tmp_path: Path,
|
||||
markdown_processor: MarkdownProcessor,
|
||||
test_config,
|
||||
):
|
||||
"""Test handling entities with no observations."""
|
||||
# Create test data
|
||||
json_path = tmp_path / "memory.json"
|
||||
test_data = [
|
||||
{
|
||||
"type": "entity",
|
||||
"name": "Empty_Entity",
|
||||
"entityType": "test",
|
||||
"observations": [] # Empty observations
|
||||
}
|
||||
]
|
||||
|
||||
with open(json_path, 'w') as f:
|
||||
for item in test_data:
|
||||
f.write(json.dumps(item) + '\n')
|
||||
|
||||
# Process import
|
||||
results = await process_memory_json(json_path, test_config.home, markdown_processor)
|
||||
|
||||
# Check results
|
||||
assert results["entities"] == 1
|
||||
assert results["relations"] == 0
|
||||
|
||||
# Verify file was created
|
||||
entity_path = test_config.home / "test/Empty_Entity.md"
|
||||
entity = await markdown_processor.read_file(entity_path)
|
||||
|
||||
assert entity.frontmatter.title == "Empty_Entity"
|
||||
assert entity.observations == []
|
||||
assert entity.relations == []
|
||||
async def test_get_markdown_processor(tmp_path, monkeypatch):
|
||||
"""Test getting markdown processor."""
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
processor = await import_memory_json.get_markdown_processor()
|
||||
assert isinstance(processor, MarkdownProcessor)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_memory_json_special_characters(
|
||||
tmp_path: Path,
|
||||
markdown_processor: MarkdownProcessor,
|
||||
test_config,
|
||||
):
|
||||
"""Test handling entities with special characters in text."""
|
||||
# Create test data
|
||||
json_path = tmp_path / "memory.json"
|
||||
test_data = [
|
||||
def test_import_json_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(import_app, ["memory-json", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_json_command_success(tmp_path, sample_json_file, monkeypatch):
|
||||
"""Test successful JSON import via command."""
|
||||
# Set up test environment
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(import_app, ["memory-json", str(sample_json_file)])
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Created 1 entities" in result.output
|
||||
assert "Added 1 relations" in result.output
|
||||
|
||||
|
||||
def test_import_json_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(import_app, ["memory-json", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_json_command_handle_old_format(tmp_path):
|
||||
"""Test handling old format JSON with from_id/to_id."""
|
||||
# Create JSON with old format
|
||||
old_format = [
|
||||
{
|
||||
"type": "entity",
|
||||
"name": "Special_Entity",
|
||||
"name": "test_entity",
|
||||
"entityType": "test",
|
||||
"observations": [
|
||||
"Contains *markdown* formatting",
|
||||
"Has #hashtags and @mentions",
|
||||
"Uses [square brackets] and {curly braces}"
|
||||
]
|
||||
}
|
||||
"observations": ["Test observation"],
|
||||
},
|
||||
{
|
||||
"type": "relation",
|
||||
"from_id": "test_entity",
|
||||
"to_id": "other_entity",
|
||||
"relation_type": "test_relation",
|
||||
},
|
||||
]
|
||||
|
||||
with open(json_path, 'w') as f:
|
||||
for item in test_data:
|
||||
f.write(json.dumps(item) + '\n')
|
||||
|
||||
# Process import
|
||||
results = await process_memory_json(json_path, test_config.home, markdown_processor)
|
||||
assert results["entities"] == 1
|
||||
|
||||
# Verify file was created and content preserved
|
||||
entity_path = test_config.home / "test/Special_Entity.md"
|
||||
entity = await markdown_processor.read_file(entity_path)
|
||||
|
||||
assert len(entity.observations) == 3
|
||||
assert entity.observations[0].content == "Contains *markdown* formatting"
|
||||
assert entity.observations[1].content == "Has #hashtags and @mentions"
|
||||
assert entity.observations[2].content == "Uses [square brackets] and {curly braces}"
|
||||
|
||||
json_file = tmp_path / "old_format.json"
|
||||
with open(json_file, "w") as f:
|
||||
for item in old_format:
|
||||
f.write(json.dumps(item) + "\n")
|
||||
|
||||
# Set up test environment
|
||||
monkeypatch = pytest.MonkeyPatch()
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(import_app, ["memory-json", str(json_file)])
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user