mirror of
https://github.com/basicmachines-co/basic-memory
synced 2026-06-21 13:47:35 +00:00
Compare commits
74 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 16466e9269 | |||
| 39bd5ca08f | |||
| 4a1f5452c4 | |||
| 9c9960e2ca | |||
| 30cd74ec95 | |||
| 132c5671d8 | |||
| 71de8acfd0 | |||
| 50469d691b | |||
| 85e620c72b | |||
| 493e0902b6 | |||
| 56f47d6812 | |||
| a15c346d5e | |||
| c06cf19e62 | |||
| 41f5b1667b | |||
| ca632beb6f | |||
| a491c2b7d4 | |||
| 674dd1fd47 | |||
| a91da13967 | |||
| e0803734e6 | |||
| 8a5a970ba9 | |||
| 65ebe5d194 | |||
| 8370a06b6c | |||
| 3270e6d82b | |||
| c05d363a99 | |||
| 3f9c68ed85 | |||
| a8ffe051ea | |||
| 2c74434e99 | |||
| bea3d6c889 | |||
| 14539010b1 | |||
| a19287c967 | |||
| 64430b850e | |||
| 10b2e91e34 | |||
| 1989db15fa | |||
| 17f0517917 | |||
| 8de84c0221 | |||
| b1d2a64933 | |||
| fe8c3d87b0 | |||
| 89ee324df0 | |||
| 8664c57bb3 | |||
| 8117a7b0ed | |||
| f47fb2aeef | |||
| 8fa197e2ec | |||
| 58f3fe95b7 | |||
| 1d6054d30a | |||
| 812947fc4b | |||
| dab957314a | |||
| 70b2eb3a4a | |||
| 80cdecdbf0 | |||
| 8dd923d5bc | |||
| 0cb429bfd3 | |||
| a15265783e | |||
| e48deaeecf | |||
| 854cf8302e | |||
| 6e4a53cdfa | |||
| 2e215fe83c | |||
| b546941779 | |||
| 29a259421a | |||
| 071e76a0ff | |||
| 9ee67e9de0 | |||
| edbc04be60 | |||
| 864209c5f8 | |||
| b40f0c75da | |||
| 230738ee9c | |||
| d15d8f91a4 | |||
| a6fdfad374 | |||
| 3e78fcc2c2 | |||
| b5d09a4854 | |||
| 01d46727b4 | |||
| 713b3bedf8 | |||
| 97d3a0196e | |||
| 052f491fff | |||
| 6589f5d251 | |||
| 9a4070eccf | |||
| 41d4d81c1a |
@@ -57,10 +57,14 @@ jobs:
|
||||
with:
|
||||
ref: ${{ needs.release.outputs.tag }}
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@v5
|
||||
- name: Set up Python "3.12"
|
||||
uses: actions/setup-python@v4
|
||||
with:
|
||||
python-version: '3.12'
|
||||
python-version: "3.12"
|
||||
cache: 'pip'
|
||||
|
||||
- name: Install librsvg
|
||||
run: brew install librsvg
|
||||
|
||||
- name: Install uv
|
||||
run: |
|
||||
@@ -70,25 +74,19 @@ jobs:
|
||||
run: |
|
||||
uv venv
|
||||
|
||||
- name: Install librsvg
|
||||
run: brew install librsvg
|
||||
|
||||
- name: Create icon
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
cd installer
|
||||
chmod +x make_icons.sh
|
||||
./make_icons.sh
|
||||
uv sync
|
||||
|
||||
- name: Build macOS installer
|
||||
run: |
|
||||
uv pip install -e ".[dev]"
|
||||
cd installer
|
||||
uv run python setup.py bdist_mac
|
||||
make installer-mac
|
||||
xattr -dr com.apple.quarantine "installer/build/Basic Memory Installer.app"
|
||||
|
||||
- name: Zip macOS installer
|
||||
run: |
|
||||
cd installer/build
|
||||
zip -r "Basic-Memory-Installer-${{ needs.release.outputs.tag }}.zip" "Basic Memory Installer.app"
|
||||
zip -ry "Basic-Memory-Installer-${{ needs.release.outputs.tag }}.zip" "Basic Memory Installer.app"
|
||||
|
||||
- name: Upload macOS installer
|
||||
uses: softprops/action-gh-release@v1
|
||||
|
||||
@@ -42,3 +42,4 @@ ENV/
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
/.coverage.*
|
||||
|
||||
+202
@@ -1,8 +1,210 @@
|
||||
# CHANGELOG
|
||||
|
||||
|
||||
## v0.4.3 (2025-02-18)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Re do enhanced read note format ([#10](https://github.com/basicmachines-co/basic-memory/pull/10),
|
||||
[`39bd5ca`](https://github.com/basicmachines-co/basic-memory/commit/39bd5ca08fd057220b95a8b5d82c5e73a1f5722b))
|
||||
|
||||
Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
|
||||
## v0.4.2 (2025-02-17)
|
||||
|
||||
|
||||
## v0.4.1 (2025-02-17)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix alemic config
|
||||
([`71de8ac`](https://github.com/basicmachines-co/basic-memory/commit/71de8acfd0902fc60f27deb3638236a3875787ab))
|
||||
|
||||
- More alembic fixes
|
||||
([`30cd74e`](https://github.com/basicmachines-co/basic-memory/commit/30cd74ec95c04eaa92b41b9815431f5fbdb46ef8))
|
||||
|
||||
|
||||
## v0.4.0 (2025-02-16)
|
||||
|
||||
### Features
|
||||
|
||||
- Import chatgpt conversation data ([#9](https://github.com/basicmachines-co/basic-memory/pull/9),
|
||||
[`56f47d6`](https://github.com/basicmachines-co/basic-memory/commit/56f47d6812982437f207629e6ac9a82e0e56514e))
|
||||
|
||||
Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
- Import claude.ai data ([#8](https://github.com/basicmachines-co/basic-memory/pull/8),
|
||||
[`a15c346`](https://github.com/basicmachines-co/basic-memory/commit/a15c346d5ebd44344b76bad877bb4d1073fcbc3b))
|
||||
|
||||
Import Claude.ai conversation and project data to basic-memory Markdown format.
|
||||
|
||||
---------
|
||||
|
||||
Co-authored-by: phernandez <phernandez@basicmachines.co>
|
||||
|
||||
|
||||
## v0.3.0 (2025-02-15)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refactor db schema migrate handling
|
||||
([`ca632be`](https://github.com/basicmachines-co/basic-memory/commit/ca632beb6fed5881f4d8ba5ce698bb5bc681e6aa))
|
||||
|
||||
|
||||
## v0.2.21 (2025-02-15)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix osx installer github action
|
||||
([`65ebe5d`](https://github.com/basicmachines-co/basic-memory/commit/65ebe5d19491e5ff047c459d799498ad5dd9cd1a))
|
||||
|
||||
- Handle memory:// url format in read_note tool
|
||||
([`e080373`](https://github.com/basicmachines-co/basic-memory/commit/e0803734e69eeb6c6d7432eea323c7a264cb8347))
|
||||
|
||||
- Remove create schema from init_db
|
||||
([`674dd1f`](https://github.com/basicmachines-co/basic-memory/commit/674dd1fd47be9e60ac17508476c62254991df288))
|
||||
|
||||
### Features
|
||||
|
||||
- Set version in var, output version at startup
|
||||
([`a91da13`](https://github.com/basicmachines-co/basic-memory/commit/a91da1396710e62587df1284da00137d156fc05e))
|
||||
|
||||
|
||||
## v0.2.20 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix installer artifact
|
||||
([`8de84c0`](https://github.com/basicmachines-co/basic-memory/commit/8de84c0221a1ee32780aa84dac4d3ea60895e05c))
|
||||
|
||||
|
||||
## v0.2.19 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Get app artifact for installer
|
||||
([`fe8c3d8`](https://github.com/basicmachines-co/basic-memory/commit/fe8c3d87b003166252290a87cbe958301cccf797))
|
||||
|
||||
|
||||
## v0.2.18 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Don't zip app on release
|
||||
([`8664c57`](https://github.com/basicmachines-co/basic-memory/commit/8664c57bb331d7f3f7e0239acb5386c7a3c6144e))
|
||||
|
||||
|
||||
## v0.2.17 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix app zip in installer release
|
||||
([`8fa197e`](https://github.com/basicmachines-co/basic-memory/commit/8fa197e2ec8a1b6caaf6dbb39c3c6626bba23e2e))
|
||||
|
||||
|
||||
## v0.2.16 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Debug inspect build on ci
|
||||
([`1d6054d`](https://github.com/basicmachines-co/basic-memory/commit/1d6054d30a477a4e6a5d6ac885632e50c01945d3))
|
||||
|
||||
|
||||
## v0.2.15 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Debug installer ci
|
||||
([`dab9573`](https://github.com/basicmachines-co/basic-memory/commit/dab957314aec9ed0e12abca2265552494ae733a2))
|
||||
|
||||
|
||||
## v0.2.14 (2025-02-14)
|
||||
|
||||
|
||||
## v0.2.13 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refactor release.yml installer
|
||||
([`a152657`](https://github.com/basicmachines-co/basic-memory/commit/a15265783e47c22d8c7931396281d023b3694e27))
|
||||
|
||||
- Try using symlinks in installer build
|
||||
([`8dd923d`](https://github.com/basicmachines-co/basic-memory/commit/8dd923d5bc0587276f92b5f1db022ad9c8687e45))
|
||||
|
||||
|
||||
## v0.2.12 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix cx_freeze options for installer
|
||||
([`854cf83`](https://github.com/basicmachines-co/basic-memory/commit/854cf8302e2f83578030db05e29b8bdc4348795a))
|
||||
|
||||
|
||||
## v0.2.11 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Ci installer app fix #37
|
||||
([`2e215fe`](https://github.com/basicmachines-co/basic-memory/commit/2e215fe83ca421b921186c7f1989dc2cb5cca278))
|
||||
|
||||
|
||||
## v0.2.10 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix build on github ci for app installer
|
||||
([`29a2594`](https://github.com/basicmachines-co/basic-memory/commit/29a259421a0ccb10cfa68e3707eaa506ad5e55c0))
|
||||
|
||||
|
||||
## v0.2.9 (2025-02-14)
|
||||
|
||||
|
||||
## v0.2.8 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Fix installer on ci, maybe
|
||||
([`edbc04b`](https://github.com/basicmachines-co/basic-memory/commit/edbc04be601d234bb1f5eb3ba24d6ad55244b031))
|
||||
|
||||
|
||||
## v0.2.7 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Try to fix installer ci
|
||||
([`230738e`](https://github.com/basicmachines-co/basic-memory/commit/230738ee9c110c0509e0a09cb0e101a92cfcb729))
|
||||
|
||||
|
||||
## v0.2.6 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Bump project patch version
|
||||
([`01d4672`](https://github.com/basicmachines-co/basic-memory/commit/01d46727b40c24b017ea9db4b741daef565ac73e))
|
||||
|
||||
- Fix installer setup.py change ci to use make
|
||||
([`3e78fcc`](https://github.com/basicmachines-co/basic-memory/commit/3e78fcc2c208d83467fe7199be17174d7ffcad1a))
|
||||
|
||||
|
||||
## v0.2.5 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Refix vitual env in installer build
|
||||
([`052f491`](https://github.com/basicmachines-co/basic-memory/commit/052f491fff629e8ead629c9259f8cb46c608d584))
|
||||
|
||||
|
||||
## v0.2.4 (2025-02-14)
|
||||
|
||||
|
||||
## v0.2.3 (2025-02-14)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- Workaround unsigned app
|
||||
([`41d4d81`](https://github.com/basicmachines-co/basic-memory/commit/41d4d81c1ad1dc2923ba0e903a57454a0c8b6b5c))
|
||||
|
||||
|
||||
## v0.2.2 (2025-02-14)
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ run-dev:
|
||||
|
||||
# Build app installer
|
||||
installer-mac:
|
||||
cd installer && chmod +x make_icons.sh && ./make_icons.sh
|
||||
cd installer && uv run python setup.py bdist_mac
|
||||
|
||||
installer-win:
|
||||
|
||||
@@ -253,6 +253,92 @@ Basic Memory is built on some key ideas:
|
||||
- Simple text patterns can capture rich meaning
|
||||
- Local-first doesn't mean feature-poor
|
||||
|
||||
## Importing data
|
||||
|
||||
Basic memory has cli commands to import data from several formats into Markdown files
|
||||
|
||||
### Claude.ai
|
||||
|
||||
First, request an export of your data from your Claude account. The data will be emailed to you in several files,
|
||||
including
|
||||
`conversations.json` and `projects.json`.
|
||||
|
||||
Import Claude.ai conversation data
|
||||
|
||||
```bash
|
||||
basic-memory import claude conversations
|
||||
```
|
||||
|
||||
The conversations will be turned into Markdown files and placed in the "conversations" folder by default (this can be
|
||||
changed with the --folder arg).
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
Importing chats from conversations.json...writing to .../basic-memory
|
||||
Reading chat data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭────────────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Imported 307 conversations │
|
||||
│ Containing 7769 messages │
|
||||
╰────────────────────────────╯
|
||||
```
|
||||
|
||||
Next, you can run the `sync` command to import the data into basic-memory
|
||||
|
||||
```bash
|
||||
basic-memory sync
|
||||
```
|
||||
|
||||
You can also import project data from Claude.ai
|
||||
|
||||
```bash
|
||||
➜ basic-memory import claude projects
|
||||
Importing projects from projects.json...writing to .../basic-memory/projects
|
||||
Reading project data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭────────────────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Imported 101 project documents │
|
||||
│ Imported 32 prompt templates │
|
||||
╰────────────────────────────────╯
|
||||
|
||||
Run 'basic-memory sync' to index the new files.
|
||||
```
|
||||
|
||||
### Chat Gpt
|
||||
|
||||
```bash
|
||||
➜ basic-memory import chatgpt
|
||||
Importing chats from conversations.json...writing to .../basic-memory/conversations
|
||||
|
||||
Reading chat data... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭────────────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Imported 198 conversations │
|
||||
│ Containing 11777 messages │
|
||||
╰────────────────────────────╯
|
||||
|
||||
|
||||
```
|
||||
|
||||
### Memory json
|
||||
|
||||
```bash
|
||||
➜ basic-memory import memory-json
|
||||
Importing from memory.json...writing to .../basic-memory
|
||||
Reading memory.json... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
Creating entities... ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 100%
|
||||
╭──────────────────────╮
|
||||
│ Import complete! │
|
||||
│ │
|
||||
│ Created 126 entities │
|
||||
│ Added 252 relations │
|
||||
╰──────────────────────╯
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
AGPL-3.0
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
# Use forward slashes (/) also on windows to provide an os agnostic path
|
||||
script_location = src/basic_memory/alembic
|
||||
|
||||
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
|
||||
# Uncomment the line below if you want the files to be prepended with date and time
|
||||
# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
|
||||
# for all available tokens
|
||||
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
|
||||
|
||||
# sys.path path, will be prepended to sys.path if present.
|
||||
# defaults to the current working directory.
|
||||
prepend_sys_path = .
|
||||
|
||||
# timezone to use when rendering the date within the migration file
|
||||
# as well as the filename.
|
||||
# If specified, requires the python>=3.9 or backports.zoneinfo library and tzdata library.
|
||||
# Any required deps can installed by adding `alembic[tz]` to the pip requirements
|
||||
# string value is passed to ZoneInfo()
|
||||
# leave blank for localtime
|
||||
# timezone =
|
||||
|
||||
# max length of characters to apply to the "slug" field
|
||||
# truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
# set to 'true' to allow .pyc and .pyo files without
|
||||
# a source .py file to be detected as revisions in the
|
||||
# versions/ directory
|
||||
# sourceless = false
|
||||
|
||||
# version location specification; This defaults
|
||||
# to migrations/versions. When using multiple version
|
||||
# directories, initial revisions must be specified with --version-path.
|
||||
# The path separator used here should be the separator specified by "version_path_separator" below.
|
||||
# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
|
||||
|
||||
# version path separator; As mentioned above, this is the character used to split
|
||||
# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
|
||||
# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
|
||||
# Valid values for version_path_separator are:
|
||||
#
|
||||
# version_path_separator = :
|
||||
# version_path_separator = ;
|
||||
# version_path_separator = space
|
||||
# version_path_separator = newline
|
||||
#
|
||||
# Use os.pathsep. Default configuration used for new projects.
|
||||
version_path_separator = os
|
||||
|
||||
# set to 'true' to search source files recursively
|
||||
# in each "version_locations" directory
|
||||
# new in Alembic version 1.10
|
||||
# recursive_version_locations = false
|
||||
|
||||
# the output encoding used when revision files
|
||||
# are written from script.py.mako
|
||||
# output_encoding = utf-8
|
||||
|
||||
sqlalchemy.url = driver://user:pass@localhost/dbname
|
||||
|
||||
|
||||
[post_write_hooks]
|
||||
# post_write_hooks defines scripts or Python functions that are run
|
||||
# on newly generated revision scripts. See the documentation for further
|
||||
# detail and examples
|
||||
|
||||
# format using "black" - use the console_scripts runner, against the "black" entrypoint
|
||||
# hooks = black
|
||||
# black.type = console_scripts
|
||||
# black.entrypoint = black
|
||||
# black.options = -l 79 REVISION_SCRIPT_FILENAME
|
||||
|
||||
# lint with attempts to fix using "ruff" - use the exec runner, execute a binary
|
||||
# hooks = ruff
|
||||
# ruff.type = exec
|
||||
# ruff.executable = %(here)s/.venv/bin/ruff
|
||||
# ruff.options = --fix REVISION_SCRIPT_FILENAME
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,26 @@
|
||||
# Basic Memory Installer
|
||||
|
||||
This installer configures Basic Memory to work with Claude Desktop.
|
||||
|
||||
## Installation
|
||||
|
||||
1. Download the latest installer from the [releases page](https://github.com/basicmachines-co/basic-memory/releases)
|
||||
2. Unzip the downloaded file
|
||||
3. Since the app is currently unsigned, you'll need to:
|
||||
|
||||
On your Mac, choose Apple menu > System Settings, then click Privacy & Security in the sidebar. (You may need to
|
||||
scroll down.)
|
||||
|
||||
Go to Security, then click Open.
|
||||
|
||||
Click Open Anyway.
|
||||
|
||||
This button is available for about an hour after you try to open the app.
|
||||
|
||||
Enter your login password, then click OK.
|
||||
|
||||
https://support.apple.com/guide/mac-help/apple-cant-check-app-for-malicious-software-mchleab3a043/mac
|
||||
|
||||
5. Restart Claude Desktop
|
||||
|
||||
The warning only appears the first time you open the app. Future updates will include proper code signing.
|
||||
@@ -8,6 +8,7 @@ if sys.platform == "darwin":
|
||||
import tkinter as tk
|
||||
from tkinter import messagebox
|
||||
|
||||
|
||||
def ensure_uv_installed():
|
||||
"""Check if uv is installed, install if not."""
|
||||
try:
|
||||
|
||||
+11
-15
@@ -4,29 +4,24 @@ import sys
|
||||
# Build options for all platforms
|
||||
build_exe_options = {
|
||||
"packages": ["json", "pathlib"],
|
||||
"excludes": [],
|
||||
"excludes": ["unittest", "pydoc", "test"],
|
||||
}
|
||||
|
||||
# Platform-specific options
|
||||
if sys.platform == "win32":
|
||||
base = "Win32GUI" # Use GUI base for Windows
|
||||
build_exe_options.update({
|
||||
"include_msvcr": True, # Include Visual C++ runtime
|
||||
})
|
||||
build_exe_options.update(
|
||||
{
|
||||
"include_msvcr": True,
|
||||
}
|
||||
)
|
||||
target_name = "Basic Memory Installer.exe"
|
||||
icon = None # We'll add Windows icon later
|
||||
else: # darwin
|
||||
base = None # Don't use GUI base for macOS
|
||||
target_name = "Basic Memory Installer"
|
||||
icon = "Basic.icns"
|
||||
|
||||
executables = [
|
||||
Executable(
|
||||
script="installer.py",
|
||||
target_name=target_name,
|
||||
base=base,
|
||||
icon=icon
|
||||
)
|
||||
Executable(script="installer.py", target_name=target_name, base=base, icon="Basic.icns")
|
||||
]
|
||||
|
||||
setup(
|
||||
@@ -37,8 +32,9 @@ setup(
|
||||
"build_exe": build_exe_options,
|
||||
"bdist_mac": {
|
||||
"bundle_name": "Basic Memory Installer",
|
||||
"iconfile": icon if sys.platform == "darwin" else None
|
||||
}
|
||||
"iconfile": "Basic.icns",
|
||||
"codesign_identity": "-", # Force ad-hoc signing
|
||||
},
|
||||
},
|
||||
executables=executables,
|
||||
)
|
||||
)
|
||||
|
||||
+4
-2
@@ -1,6 +1,6 @@
|
||||
[project]
|
||||
name = "basic-memory"
|
||||
version = "0.2.3"
|
||||
version = "0.4.3"
|
||||
description = "Local-first knowledge management combining Zettelkasten with knowledge graphs"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.12.1"
|
||||
@@ -84,7 +84,9 @@ pythonVersion = "3.12"
|
||||
|
||||
|
||||
[tool.semantic_release]
|
||||
version_variable = "src/basic_memory/__init__.py:__version__"
|
||||
version_variables = [
|
||||
"src/basic_memory/__init__.py:__version__",
|
||||
]
|
||||
version_toml = [
|
||||
"pyproject.toml:project.version",
|
||||
]
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
"""basic-memory - Local-first knowledge management combining Zettelkasten with knowledge graphs"""
|
||||
|
||||
__version__ = "0.0.1"
|
||||
__version__ = "0.4.3"
|
||||
|
||||
@@ -6,38 +6,17 @@ from fastapi import FastAPI, HTTPException
|
||||
from fastapi.exception_handlers import http_exception_handler
|
||||
from loguru import logger
|
||||
|
||||
import basic_memory
|
||||
from basic_memory import db
|
||||
from basic_memory.config import config as app_config
|
||||
from basic_memory.api.routers import knowledge, search, memory, resource
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.repository.search_repository import SearchRepository
|
||||
|
||||
|
||||
async def run_migrations(): # pragma: no cover
|
||||
"""Run any pending alembic migrations."""
|
||||
logger.info("Running database migrations...")
|
||||
try:
|
||||
config = Config("alembic.ini")
|
||||
command.upgrade(config, "head")
|
||||
logger.info("Migrations completed successfully")
|
||||
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
app_config.database_path, DatabaseType.FILESYSTEM
|
||||
)
|
||||
await SearchRepository(session_maker).init_search_index()
|
||||
except Exception as e:
|
||||
logger.error(f"Error running migrations: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI): # pragma: no cover
|
||||
"""Lifecycle manager for the FastAPI app."""
|
||||
logger.info("Starting Basic Memory API")
|
||||
await run_migrations()
|
||||
logger.info(f"Starting Basic Memory API {basic_memory.__version__}")
|
||||
await db.run_migrations(app_config)
|
||||
yield
|
||||
logger.info("Shutting down Basic Memory API")
|
||||
await db.shutdown_db()
|
||||
|
||||
@@ -1,34 +1,118 @@
|
||||
"""Routes for getting entity content."""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi import APIRouter, HTTPException, BackgroundTasks
|
||||
from fastapi.responses import FileResponse
|
||||
from loguru import logger
|
||||
|
||||
from basic_memory.deps import ProjectConfigDep, LinkResolverDep
|
||||
from basic_memory.deps import (
|
||||
ProjectConfigDep,
|
||||
LinkResolverDep,
|
||||
SearchServiceDep,
|
||||
EntityServiceDep,
|
||||
FileServiceDep,
|
||||
)
|
||||
from basic_memory.repository.search_repository import SearchIndexRow
|
||||
from basic_memory.schemas.memory import normalize_memory_url
|
||||
from basic_memory.schemas.search import SearchQuery, SearchItemType
|
||||
|
||||
router = APIRouter(prefix="/resource", tags=["resources"])
|
||||
|
||||
|
||||
def get_entity_ids(item: SearchIndexRow) -> list[int]:
|
||||
match item.type:
|
||||
case SearchItemType.ENTITY:
|
||||
return [item.id]
|
||||
case SearchItemType.OBSERVATION:
|
||||
return [item.entity_id] # pyright: ignore [reportReturnType]
|
||||
case SearchItemType.RELATION:
|
||||
from_entity = item.from_id
|
||||
to_entity = item.to_id # pyright: ignore [reportReturnType]
|
||||
return [from_entity, to_entity] if to_entity else [from_entity] # pyright: ignore [reportReturnType]
|
||||
case _:
|
||||
raise ValueError(f"Unexpected type: {item.type}")
|
||||
|
||||
|
||||
@router.get("/{identifier:path}")
|
||||
async def get_resource_content(
|
||||
config: ProjectConfigDep,
|
||||
link_resolver: LinkResolverDep,
|
||||
search_service: SearchServiceDep,
|
||||
entity_service: EntityServiceDep,
|
||||
file_service: FileServiceDep,
|
||||
background_tasks: BackgroundTasks,
|
||||
identifier: str,
|
||||
) -> FileResponse:
|
||||
"""Get resource content by identifier: name or permalink."""
|
||||
logger.debug(f"Getting content for permalink: {identifier}")
|
||||
logger.debug(f"Getting content for: {identifier}")
|
||||
|
||||
# Find entity by permalink
|
||||
# Find single entity by permalink
|
||||
entity = await link_resolver.resolve_link(identifier)
|
||||
if not entity:
|
||||
raise HTTPException(status_code=404, detail=f"Entity not found: {identifier}")
|
||||
results = [entity] if entity else []
|
||||
|
||||
file_path = Path(f"{config.home}/{entity.file_path}")
|
||||
if not file_path.exists():
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"File not found: {file_path}",
|
||||
# search using the identifier as a permalink
|
||||
if not results:
|
||||
# if the identifier contains a wildcard, use GLOB search
|
||||
query = (
|
||||
SearchQuery(permalink_match=identifier)
|
||||
if "*" in identifier
|
||||
else SearchQuery(permalink=identifier)
|
||||
)
|
||||
return FileResponse(path=file_path)
|
||||
search_results = await search_service.search(query)
|
||||
if not search_results:
|
||||
raise HTTPException(status_code=404, detail=f"Resource not found: {identifier}")
|
||||
|
||||
# get the entities related to the search results
|
||||
entity_ids = [id for result in search_results for id in get_entity_ids(result)]
|
||||
results = await entity_service.get_entities_by_id(entity_ids)
|
||||
|
||||
# return single response
|
||||
if len(results) == 1:
|
||||
entity = results[0]
|
||||
file_path = Path(f"{config.home}/{entity.file_path}")
|
||||
if not file_path.exists():
|
||||
raise HTTPException(
|
||||
status_code=404,
|
||||
detail=f"File not found: {file_path}",
|
||||
)
|
||||
return FileResponse(path=file_path)
|
||||
|
||||
# for multiple files, initialize a temporary file for writing the results
|
||||
with tempfile.NamedTemporaryFile(delete=False, mode="w", suffix=".md") as tmp_file:
|
||||
temp_file_path = tmp_file.name
|
||||
|
||||
for result in results:
|
||||
# Read content for each entity
|
||||
content = await file_service.read_entity_content(result)
|
||||
memory_url = normalize_memory_url(result.permalink)
|
||||
modified_date = result.updated_at.isoformat()
|
||||
assert result.checksum
|
||||
checksum = result.checksum[:8]
|
||||
|
||||
# Prepare the delimited content
|
||||
response_content = f"--- {memory_url} {modified_date} {checksum}\n"
|
||||
response_content += f"\n{content}\n"
|
||||
response_content += "\n"
|
||||
|
||||
# Write content directly to the temporary file in append mode
|
||||
tmp_file.write(response_content)
|
||||
|
||||
# Ensure all content is written to disk
|
||||
tmp_file.flush()
|
||||
|
||||
# Schedule the temporary file to be deleted after the response
|
||||
background_tasks.add_task(cleanup_temp_file, temp_file_path)
|
||||
|
||||
# Return the file response
|
||||
return FileResponse(path=temp_file_path)
|
||||
|
||||
|
||||
def cleanup_temp_file(file_path: str):
|
||||
"""Delete the temporary file."""
|
||||
try:
|
||||
Path(file_path).unlink() # Deletes the file
|
||||
logger.debug(f"Temporary file deleted: {file_path}")
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error deleting temporary file {file_path}: {e}")
|
||||
|
||||
@@ -1,3 +1,20 @@
|
||||
import asyncio
|
||||
|
||||
import typer
|
||||
|
||||
app = typer.Typer()
|
||||
from basic_memory import db
|
||||
from basic_memory.config import config
|
||||
from basic_memory.utils import setup_logging
|
||||
|
||||
setup_logging(log_file=".basic-memory/basic-memory-cli.log") # pragma: no cover
|
||||
|
||||
asyncio.run(db.run_migrations(config))
|
||||
|
||||
app = typer.Typer(name="basic-memory")
|
||||
|
||||
import_app = typer.Typer()
|
||||
app.add_typer(import_app, name="import")
|
||||
|
||||
|
||||
claude_app = typer.Typer()
|
||||
import_app.add_typer(claude_app, name="claude")
|
||||
|
||||
@@ -22,4 +22,4 @@ def reset(
|
||||
from basic_memory.cli.commands.sync import sync
|
||||
|
||||
logger.info("Rebuilding search index from filesystem...")
|
||||
asyncio.run(sync()) # pyright: ignore
|
||||
sync(watch=False) # pyright: ignore
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
"""Import command for ChatGPT conversations."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated, Set, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_timestamp(ts: float) -> str:
|
||||
"""Format Unix timestamp for display."""
|
||||
dt = datetime.fromtimestamp(ts)
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def get_message_content(message: Dict[str, Any]) -> str:
|
||||
"""Extract clean message content."""
|
||||
if not message or "content" not in message:
|
||||
return "" # pragma: no cover
|
||||
|
||||
content = message["content"]
|
||||
if content.get("content_type") == "text":
|
||||
return "\n".join(content.get("parts", []))
|
||||
elif content.get("content_type") == "code":
|
||||
return f"```{content.get('language', '')}\n{content.get('text', '')}\n```"
|
||||
return "" # pragma: no cover
|
||||
|
||||
|
||||
def traverse_messages(
|
||||
mapping: Dict[str, Any], root_id: Optional[str], seen: Set[str]
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Traverse message tree and return messages in order."""
|
||||
messages = []
|
||||
node = mapping.get(root_id) if root_id else None
|
||||
|
||||
while node:
|
||||
if node["id"] not in seen and node.get("message"):
|
||||
seen.add(node["id"])
|
||||
messages.append(node["message"])
|
||||
|
||||
# Follow children
|
||||
children = node.get("children", [])
|
||||
for child_id in children:
|
||||
child_msgs = traverse_messages(mapping, child_id, seen)
|
||||
messages.extend(child_msgs)
|
||||
|
||||
break # Don't follow siblings
|
||||
|
||||
return messages
|
||||
|
||||
|
||||
def format_chat_markdown(
|
||||
title: str,
|
||||
mapping: Dict[str, Any],
|
||||
root_id: Optional[str],
|
||||
created_at: float,
|
||||
modified_at: float,
|
||||
) -> str:
|
||||
"""Format chat as clean markdown."""
|
||||
|
||||
# Start with title
|
||||
lines = [f"# {title}\n"]
|
||||
|
||||
# Traverse message tree
|
||||
seen_msgs = set()
|
||||
messages = traverse_messages(mapping, root_id, seen_msgs)
|
||||
|
||||
# Format each message
|
||||
for msg in messages:
|
||||
# Skip hidden messages
|
||||
if msg.get("metadata", {}).get("is_visually_hidden_from_conversation"):
|
||||
continue
|
||||
|
||||
# Get author and timestamp
|
||||
author = msg["author"]["role"].title()
|
||||
ts = format_timestamp(msg["create_time"]) if msg.get("create_time") else ""
|
||||
|
||||
# Add message header
|
||||
lines.append(f"### {author} ({ts})")
|
||||
|
||||
# Add message content
|
||||
content = get_message_content(msg)
|
||||
if content:
|
||||
lines.append(content)
|
||||
|
||||
# Add spacing
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_chat_content(folder: str, conversation: Dict[str, Any]) -> EntityMarkdown:
|
||||
"""Convert chat conversation to Basic Memory entity."""
|
||||
|
||||
# Extract timestamps
|
||||
created_at = conversation["create_time"]
|
||||
modified_at = conversation["update_time"]
|
||||
|
||||
root_id = None
|
||||
# Find root message
|
||||
for node_id, node in conversation["mapping"].items():
|
||||
if node.get("parent") is None:
|
||||
root_id = node_id
|
||||
break
|
||||
|
||||
# Generate permalink
|
||||
date_prefix = datetime.fromtimestamp(created_at).strftime("%Y%m%d")
|
||||
clean_title = clean_filename(conversation["title"])
|
||||
|
||||
# Format content
|
||||
content = format_chat_markdown(
|
||||
title=conversation["title"],
|
||||
mapping=conversation["mapping"],
|
||||
root_id=root_id,
|
||||
created_at=created_at,
|
||||
modified_at=modified_at,
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "conversation",
|
||||
"title": conversation["title"],
|
||||
"created": format_timestamp(created_at),
|
||||
"modified": format_timestamp(modified_at),
|
||||
"permalink": f"{folder}/{date_prefix}-{clean_title}",
|
||||
}
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_chatgpt_json(
|
||||
json_path: Path, folder: str, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import conversations from ChatGPT JSON format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading chat data...", total=None)
|
||||
|
||||
# Read conversations
|
||||
conversations = json.loads(json_path.read_text())
|
||||
progress.update(read_task, total=len(conversations))
|
||||
|
||||
# Process each conversation
|
||||
messages_imported = 0
|
||||
chats_imported = 0
|
||||
|
||||
for chat in conversations:
|
||||
# Convert to entity
|
||||
entity = format_chat_content(folder, chat)
|
||||
|
||||
# Write file
|
||||
file_path = config.home / f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
# logger.info(f"Writing file: {file_path.absolute()}")
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
|
||||
# Count messages
|
||||
msg_count = sum(
|
||||
1
|
||||
for node in chat["mapping"].values()
|
||||
if node.get("message")
|
||||
and not node.get("message", {})
|
||||
.get("metadata", {})
|
||||
.get("is_visually_hidden_from_conversation")
|
||||
)
|
||||
|
||||
chats_imported += 1
|
||||
messages_imported += msg_count
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"conversations": chats_imported, "messages": messages_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@import_app.command(name="chatgpt", help="Import conversations from ChatGPT JSON export.")
|
||||
def import_chatgpt(
|
||||
conversations_json: Annotated[
|
||||
Path, typer.Option(..., help="Path to ChatGPT conversations.json file")
|
||||
] = Path("conversations.json"),
|
||||
folder: Annotated[
|
||||
str, typer.Option(help="The folder to place the files in.")
|
||||
] = "conversations",
|
||||
):
|
||||
"""Import chat conversations from ChatGPT JSON format.
|
||||
|
||||
This command will:
|
||||
1. Read the complex tree structure of messages
|
||||
2. Convert them to linear markdown conversations
|
||||
3. Save as clean, readable markdown files
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if conversations_json:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_chatgpt_json(conversations_json, folder, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,211 @@
|
||||
"""Import command for basic-memory CLI to import chat data from conversations2.json format."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import claude_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
# Remove invalid characters and convert spaces
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_timestamp(ts: str) -> str:
|
||||
"""Format ISO timestamp for display."""
|
||||
dt = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
||||
return dt.strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
def format_chat_markdown(
|
||||
name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str, permalink: str
|
||||
) -> str:
|
||||
"""Format chat as clean markdown."""
|
||||
|
||||
# Start with frontmatter and title
|
||||
lines = [
|
||||
f"# {name}\n",
|
||||
]
|
||||
|
||||
# Add messages
|
||||
for msg in messages:
|
||||
# Format timestamp
|
||||
ts = format_timestamp(msg["created_at"])
|
||||
|
||||
# Add message header
|
||||
lines.append(f"### {msg['sender'].title()} ({ts})")
|
||||
|
||||
# Handle message content
|
||||
content = msg.get("text", "")
|
||||
if msg.get("content"):
|
||||
content = " ".join(c.get("text", "") for c in msg["content"])
|
||||
lines.append(content)
|
||||
|
||||
# Handle attachments
|
||||
attachments = msg.get("attachments", [])
|
||||
for attachment in attachments:
|
||||
if "file_name" in attachment:
|
||||
lines.append(f"\n**Attachment: {attachment['file_name']}**")
|
||||
if "extracted_content" in attachment:
|
||||
lines.append("```")
|
||||
lines.append(attachment["extracted_content"])
|
||||
lines.append("```")
|
||||
|
||||
# Add spacing between messages
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def format_chat_content(
|
||||
base_path: Path, name: str, messages: List[Dict[str, Any]], created_at: str, modified_at: str
|
||||
) -> EntityMarkdown:
|
||||
"""Convert chat messages to Basic Memory entity format."""
|
||||
|
||||
# Generate permalink
|
||||
date_prefix = datetime.fromisoformat(created_at.replace("Z", "+00:00")).strftime("%Y%m%d")
|
||||
clean_title = clean_filename(name)
|
||||
permalink = f"{base_path}/{date_prefix}-{clean_title}"
|
||||
|
||||
# Format content
|
||||
content = format_chat_markdown(
|
||||
name=name,
|
||||
messages=messages,
|
||||
created_at=created_at,
|
||||
modified_at=modified_at,
|
||||
permalink=permalink,
|
||||
)
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "conversation",
|
||||
"title": name,
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": permalink,
|
||||
}
|
||||
),
|
||||
content=content,
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_conversations_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import chat data from conversations2.json format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading chat data...", total=None)
|
||||
|
||||
# Read chat data - handle array of arrays format
|
||||
data = json.loads(json_path.read_text())
|
||||
conversations = [chat for chat in data]
|
||||
progress.update(read_task, total=len(conversations))
|
||||
|
||||
# Process each conversation
|
||||
messages_imported = 0
|
||||
chats_imported = 0
|
||||
|
||||
for chat in conversations:
|
||||
# Convert to entity
|
||||
entity = format_chat_content(
|
||||
base_path=base_path,
|
||||
name=chat["name"],
|
||||
messages=chat["chat_messages"],
|
||||
created_at=chat["created_at"],
|
||||
modified_at=chat["updated_at"],
|
||||
)
|
||||
|
||||
# Write file
|
||||
file_path = Path(f"{entity.frontmatter.metadata['permalink']}.md")
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
|
||||
chats_imported += 1
|
||||
messages_imported += len(chat["chat_messages"])
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"conversations": chats_imported, "messages": messages_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@claude_app.command(name="conversations", help="Import chat conversations from Claude.ai.")
|
||||
def import_claude(
|
||||
conversations_json: Annotated[
|
||||
Path, typer.Argument(..., help="Path to conversations.json file")
|
||||
] = Path("conversations.json"),
|
||||
folder: Annotated[
|
||||
str, typer.Option(help="The folder to place the files in.")
|
||||
] = "conversations",
|
||||
):
|
||||
"""Import chat conversations from conversations2.json format.
|
||||
|
||||
This command will:
|
||||
1. Read chat data and nested messages
|
||||
2. Create markdown files for each conversation
|
||||
3. Format content in clean, readable markdown
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if not conversations_json.exists():
|
||||
typer.echo(f"Error: File not found: {conversations_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / folder
|
||||
console.print(f"\nImporting chats from {conversations_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_conversations_json(conversations_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['conversations']} conversations\n"
|
||||
f"Containing {results['messages']} messages",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Import command for basic-memory CLI to import project data from Claude.ai."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, Annotated, Optional
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import claude_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
def clean_filename(text: str) -> str:
|
||||
"""Convert text to safe filename."""
|
||||
clean = "".join(c if c.isalnum() else "-" for c in text.lower()).strip("-")
|
||||
return clean
|
||||
|
||||
|
||||
def format_project_markdown(project: Dict[str, Any], doc: Dict[str, Any]) -> EntityMarkdown:
|
||||
"""Format a project document as a Basic Memory entity."""
|
||||
|
||||
# Extract timestamps
|
||||
created_at = doc.get("created_at") or project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean names for organization
|
||||
project_dir = clean_filename(project["name"])
|
||||
doc_file = clean_filename(doc["filename"])
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "project_doc",
|
||||
"title": doc["filename"],
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": f"{project_dir}/docs/{doc_file}",
|
||||
"project_name": project["name"],
|
||||
"project_uuid": project["uuid"],
|
||||
"doc_uuid": doc["uuid"],
|
||||
}
|
||||
),
|
||||
content=doc["content"],
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
def format_prompt_markdown(project: Dict[str, Any]) -> Optional[EntityMarkdown]:
|
||||
"""Format project prompt template as a Basic Memory entity."""
|
||||
|
||||
if not project.get("prompt_template"):
|
||||
return None
|
||||
|
||||
# Extract timestamps
|
||||
created_at = project["created_at"]
|
||||
modified_at = project["updated_at"]
|
||||
|
||||
# Generate clean project directory name
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Create entity
|
||||
entity = EntityMarkdown(
|
||||
frontmatter=EntityFrontmatter(
|
||||
metadata={
|
||||
"type": "prompt_template",
|
||||
"title": f"Prompt Template: {project['name']}",
|
||||
"created": created_at,
|
||||
"modified": modified_at,
|
||||
"permalink": f"{project_dir}/prompt-template",
|
||||
"project_name": project["name"],
|
||||
"project_uuid": project["uuid"],
|
||||
}
|
||||
),
|
||||
content=f"# Prompt Template: {project['name']}\n\n{project['prompt_template']}",
|
||||
)
|
||||
|
||||
return entity
|
||||
|
||||
|
||||
async def process_projects_json(
|
||||
json_path: Path, base_path: Path, markdown_processor: MarkdownProcessor
|
||||
) -> Dict[str, int]:
|
||||
"""Import project data from Claude.ai projects.json format."""
|
||||
|
||||
with Progress(
|
||||
SpinnerColumn(),
|
||||
TextColumn("[progress.description]{task.description}"),
|
||||
BarColumn(),
|
||||
TextColumn("[progress.percentage]{task.percentage:>3.0f}%"),
|
||||
console=console,
|
||||
) as progress:
|
||||
read_task = progress.add_task("Reading project data...", total=None)
|
||||
|
||||
# Read project data
|
||||
data = json.loads(json_path.read_text())
|
||||
progress.update(read_task, total=len(data))
|
||||
|
||||
# Track import counts
|
||||
docs_imported = 0
|
||||
prompts_imported = 0
|
||||
|
||||
# Process each project
|
||||
for project in data:
|
||||
project_dir = clean_filename(project["name"])
|
||||
|
||||
# Create project directories
|
||||
docs_dir = base_path / project_dir / "docs"
|
||||
docs_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Import prompt template if it exists
|
||||
if prompt_entity := format_prompt_markdown(project):
|
||||
file_path = base_path / f"{prompt_entity.frontmatter.metadata['permalink']}.md"
|
||||
await markdown_processor.write_file(file_path, prompt_entity)
|
||||
prompts_imported += 1
|
||||
|
||||
# Import project documents
|
||||
for doc in project.get("docs", []):
|
||||
entity = format_project_markdown(project, doc)
|
||||
file_path = base_path / f"{entity.frontmatter.metadata['permalink']}.md"
|
||||
await markdown_processor.write_file(file_path, entity)
|
||||
docs_imported += 1
|
||||
|
||||
progress.update(read_task, advance=1)
|
||||
|
||||
return {"documents": docs_imported, "prompts": prompts_imported}
|
||||
|
||||
|
||||
async def get_markdown_processor() -> MarkdownProcessor:
|
||||
"""Get MarkdownProcessor instance."""
|
||||
entity_parser = EntityParser(config.home)
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@claude_app.command(name="projects", help="Import projects from Claude.ai.")
|
||||
def import_projects(
|
||||
projects_json: Annotated[Path, typer.Argument(..., help="Path to projects.json file")] = Path(
|
||||
"projects.json"
|
||||
),
|
||||
base_folder: Annotated[
|
||||
str, typer.Option(help="The base folder to place project files in.")
|
||||
] = "projects",
|
||||
):
|
||||
"""Import project data from Claude.ai.
|
||||
|
||||
This command will:
|
||||
1. Create a directory for each project
|
||||
2. Store docs in a docs/ subdirectory
|
||||
3. Place prompt template in project root
|
||||
|
||||
After importing, run 'basic-memory sync' to index the new files.
|
||||
"""
|
||||
|
||||
try:
|
||||
if projects_json:
|
||||
if not projects_json.exists():
|
||||
typer.echo(f"Error: File not found: {projects_json}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Get markdown processor
|
||||
markdown_processor = asyncio.run(get_markdown_processor())
|
||||
|
||||
# Process the file
|
||||
base_path = config.home / base_folder if base_folder else config.home
|
||||
console.print(f"\nImporting projects from {projects_json}...writing to {base_path}")
|
||||
results = asyncio.run(
|
||||
process_projects_json(projects_json, base_path, markdown_processor)
|
||||
)
|
||||
|
||||
# Show results
|
||||
console.print(
|
||||
Panel(
|
||||
f"[green]Import complete![/green]\n\n"
|
||||
f"Imported {results['documents']} project documents\n"
|
||||
f"Imported {results['prompts']} prompt templates",
|
||||
expand=False,
|
||||
)
|
||||
)
|
||||
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
@@ -3,7 +3,7 @@
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, Any, List
|
||||
from typing import Dict, Any, List, Annotated
|
||||
|
||||
import typer
|
||||
from loguru import logger
|
||||
@@ -11,12 +11,11 @@ from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
from basic_memory.markdown.schemas import EntityMarkdown, EntityFrontmatter, Observation, Relation
|
||||
|
||||
|
||||
console = Console()
|
||||
|
||||
|
||||
@@ -98,9 +97,11 @@ async def get_markdown_processor() -> MarkdownProcessor:
|
||||
return MarkdownProcessor(entity_parser)
|
||||
|
||||
|
||||
@app.command()
|
||||
def import_json(
|
||||
json_path: Path = typer.Argument(..., help="Path to memory.json file to import"),
|
||||
@import_app.command()
|
||||
def memory_json(
|
||||
json_path: Annotated[Path, typer.Argument(..., help="Path to memory.json file")] = Path(
|
||||
"memory.json"
|
||||
),
|
||||
):
|
||||
"""Import entities and relations from a memory.json file.
|
||||
|
||||
@@ -138,6 +139,6 @@ def import_json(
|
||||
console.print("\nRun 'basic-memory sync' to index the new files.")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Import failed")
|
||||
logger.error("Import failed")
|
||||
typer.echo(f"Error during import: {e}", err=True)
|
||||
raise typer.Exit(1)
|
||||
|
||||
@@ -12,9 +12,9 @@ import basic_memory.mcp.tools # noqa: F401 # pragma: no cover
|
||||
|
||||
|
||||
@app.command()
|
||||
def mcp():
|
||||
def mcp(): # pragma: no cover
|
||||
"""Run the MCP server for Claude Desktop integration."""
|
||||
home_dir = config.home
|
||||
logger.info("Starting Basic Memory MCP server")
|
||||
logger.info(f"Starting Basic Memory MCP server {basic_memory.__version__}")
|
||||
logger.info(f"Home directory: {home_dir}")
|
||||
mcp_server.run()
|
||||
|
||||
@@ -25,13 +25,11 @@ async def get_file_change_scanner(
|
||||
db_type=DatabaseType.FILESYSTEM,
|
||||
) -> FileChangeScanner: # pragma: no cover
|
||||
"""Get sync service instance."""
|
||||
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
|
||||
engine,
|
||||
session_maker,
|
||||
):
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
return file_change_scanner
|
||||
_, session_maker = await db.get_or_create_db(db_path=config.database_path, db_type=db_type)
|
||||
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
return file_change_scanner
|
||||
|
||||
|
||||
def add_files_to_tree(
|
||||
|
||||
@@ -14,7 +14,6 @@ from rich.tree import Tree
|
||||
from basic_memory import db
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.config import config
|
||||
from basic_memory.db import DatabaseType
|
||||
from basic_memory.markdown import EntityParser
|
||||
from basic_memory.markdown.markdown_processor import MarkdownProcessor
|
||||
from basic_memory.repository import (
|
||||
@@ -39,50 +38,50 @@ class ValidationIssue:
|
||||
error: str
|
||||
|
||||
|
||||
async def get_sync_service(db_type=DatabaseType.FILESYSTEM): # pragma: no cover
|
||||
async def get_sync_service(): # pragma: no cover
|
||||
"""Get sync service instance with all dependencies."""
|
||||
async with db.engine_session_factory(db_path=config.database_path, db_type=db_type) as (
|
||||
engine,
|
||||
session_maker,
|
||||
):
|
||||
entity_parser = EntityParser(config.home)
|
||||
markdown_processor = MarkdownProcessor(entity_parser)
|
||||
file_service = FileService(config.home, markdown_processor)
|
||||
_, session_maker = await db.get_or_create_db(
|
||||
db_path=config.database_path, db_type=db.DatabaseType.FILESYSTEM
|
||||
)
|
||||
|
||||
# Initialize repositories
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
observation_repository = ObservationRepository(session_maker)
|
||||
relation_repository = RelationRepository(session_maker)
|
||||
search_repository = SearchRepository(session_maker)
|
||||
entity_parser = EntityParser(config.home)
|
||||
markdown_processor = MarkdownProcessor(entity_parser)
|
||||
file_service = FileService(config.home, markdown_processor)
|
||||
|
||||
# Initialize services
|
||||
search_service = SearchService(search_repository, entity_repository, file_service)
|
||||
link_resolver = LinkResolver(entity_repository, search_service)
|
||||
# Initialize repositories
|
||||
entity_repository = EntityRepository(session_maker)
|
||||
observation_repository = ObservationRepository(session_maker)
|
||||
relation_repository = RelationRepository(session_maker)
|
||||
search_repository = SearchRepository(session_maker)
|
||||
|
||||
# Initialize scanner
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
# Initialize services
|
||||
search_service = SearchService(search_repository, entity_repository, file_service)
|
||||
link_resolver = LinkResolver(entity_repository, search_service)
|
||||
|
||||
# Initialize services
|
||||
entity_service = EntityService(
|
||||
entity_parser,
|
||||
entity_repository,
|
||||
observation_repository,
|
||||
relation_repository,
|
||||
file_service,
|
||||
link_resolver,
|
||||
)
|
||||
# Initialize scanner
|
||||
file_change_scanner = FileChangeScanner(entity_repository)
|
||||
|
||||
# Create sync service
|
||||
sync_service = SyncService(
|
||||
scanner=file_change_scanner,
|
||||
entity_service=entity_service,
|
||||
entity_parser=entity_parser,
|
||||
entity_repository=entity_repository,
|
||||
relation_repository=relation_repository,
|
||||
search_service=search_service,
|
||||
)
|
||||
# Initialize services
|
||||
entity_service = EntityService(
|
||||
entity_parser,
|
||||
entity_repository,
|
||||
observation_repository,
|
||||
relation_repository,
|
||||
file_service,
|
||||
link_resolver,
|
||||
)
|
||||
|
||||
return sync_service
|
||||
# Create sync service
|
||||
sync_service = SyncService(
|
||||
scanner=file_change_scanner,
|
||||
entity_service=entity_service,
|
||||
entity_parser=entity_parser,
|
||||
entity_repository=entity_repository,
|
||||
relation_repository=relation_repository,
|
||||
search_service=search_service,
|
||||
)
|
||||
|
||||
return sync_service
|
||||
|
||||
|
||||
def group_issues_by_directory(issues: List[ValidationIssue]) -> Dict[str, List[ValidationIssue]]:
|
||||
@@ -154,6 +153,7 @@ def display_detailed_sync_results(knowledge: SyncReport):
|
||||
|
||||
async def run_sync(verbose: bool = False, watch: bool = False):
|
||||
"""Run sync operation."""
|
||||
|
||||
sync_service = await get_sync_service()
|
||||
|
||||
# Start watching if requested
|
||||
|
||||
@@ -4,7 +4,16 @@ from basic_memory.cli.app import app # pragma: no cover
|
||||
from basic_memory.utils import setup_logging # pragma: no cover
|
||||
|
||||
# Register commands
|
||||
from basic_memory.cli.commands import status, sync, db, import_memory_json, mcp # noqa: F401 # pragma: no cover
|
||||
from basic_memory.cli.commands import ( # noqa: F401 # pragma: no cover
|
||||
status,
|
||||
sync,
|
||||
db,
|
||||
import_memory_json,
|
||||
mcp,
|
||||
import_claude_conversations,
|
||||
import_claude_projects,
|
||||
import_chatgpt,
|
||||
)
|
||||
|
||||
|
||||
# Set up logging when module is imported
|
||||
|
||||
+36
-28
@@ -4,6 +4,10 @@ from enum import Enum, auto
|
||||
from pathlib import Path
|
||||
from typing import AsyncGenerator, Optional
|
||||
|
||||
from basic_memory.config import ProjectConfig
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
@@ -14,8 +18,7 @@ from sqlalchemy.ext.asyncio import (
|
||||
async_scoped_session,
|
||||
)
|
||||
|
||||
from basic_memory.models import Base
|
||||
from basic_memory.models.search import CREATE_SEARCH_INDEX
|
||||
from basic_memory.repository.search_repository import SearchRepository
|
||||
|
||||
# Module level state
|
||||
_engine: Optional[AsyncEngine] = None
|
||||
@@ -35,7 +38,7 @@ class DatabaseType(Enum):
|
||||
logger.info("Using in-memory SQLite database")
|
||||
return "sqlite+aiosqlite://"
|
||||
|
||||
return f"sqlite+aiosqlite:///{db_path}"
|
||||
return f"sqlite+aiosqlite:///{db_path}" # pragma: no cover
|
||||
|
||||
|
||||
def get_scoped_session_factory(
|
||||
@@ -69,24 +72,6 @@ async def scoped_session(
|
||||
await factory.remove()
|
||||
|
||||
|
||||
async def init_db() -> None:
|
||||
"""Initialize database with required tables."""
|
||||
if _session_maker is None: # pragma: no cover
|
||||
raise RuntimeError("Database session maker not initialized")
|
||||
|
||||
logger.info("Initializing database...")
|
||||
|
||||
async with scoped_session(_session_maker) as session:
|
||||
await session.execute(text("PRAGMA foreign_keys=ON"))
|
||||
conn = await session.connection()
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
# recreate search index
|
||||
await session.execute(CREATE_SEARCH_INDEX)
|
||||
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def get_or_create_db(
|
||||
db_path: Path,
|
||||
db_type: DatabaseType = DatabaseType.FILESYSTEM,
|
||||
@@ -100,9 +85,6 @@ async def get_or_create_db(
|
||||
_engine = create_async_engine(db_url, connect_args={"check_same_thread": False})
|
||||
_session_maker = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
# Initialize database
|
||||
await init_db()
|
||||
|
||||
assert _engine is not None # for type checker
|
||||
assert _session_maker is not None # for type checker
|
||||
return _engine, _session_maker
|
||||
@@ -122,7 +104,6 @@ async def shutdown_db() -> None: # pragma: no cover
|
||||
async def engine_session_factory(
|
||||
db_path: Path,
|
||||
db_type: DatabaseType = DatabaseType.MEMORY,
|
||||
init: bool = True,
|
||||
) -> AsyncGenerator[tuple[AsyncEngine, async_sessionmaker[AsyncSession]], None]:
|
||||
"""Create engine and session factory.
|
||||
|
||||
@@ -139,9 +120,6 @@ async def engine_session_factory(
|
||||
try:
|
||||
_session_maker = async_sessionmaker(_engine, expire_on_commit=False)
|
||||
|
||||
if init:
|
||||
await init_db()
|
||||
|
||||
assert _engine is not None # for type checker
|
||||
assert _session_maker is not None # for type checker
|
||||
yield _engine, _session_maker
|
||||
@@ -150,3 +128,33 @@ async def engine_session_factory(
|
||||
await _engine.dispose()
|
||||
_engine = None
|
||||
_session_maker = None
|
||||
|
||||
|
||||
async def run_migrations(app_config: ProjectConfig, database_type=DatabaseType.FILESYSTEM):
|
||||
"""Run any pending alembic migrations."""
|
||||
logger.info("Running database migrations...")
|
||||
try:
|
||||
# Get the absolute path to the alembic directory relative to this file
|
||||
alembic_dir = Path(__file__).parent / "alembic"
|
||||
config = Config()
|
||||
|
||||
# Set required Alembic config options programmatically
|
||||
config.set_main_option("script_location", str(alembic_dir))
|
||||
config.set_main_option(
|
||||
"file_template",
|
||||
"%%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s",
|
||||
)
|
||||
config.set_main_option("timezone", "UTC")
|
||||
config.set_main_option("revision_environment", "false")
|
||||
config.set_main_option(
|
||||
"sqlalchemy.url", DatabaseType.get_db_url(app_config.database_path, database_type)
|
||||
)
|
||||
|
||||
command.upgrade(config, "head")
|
||||
logger.info("Migrations completed successfully")
|
||||
|
||||
_, session_maker = await get_or_create_db(app_config.database_path, database_type)
|
||||
await SearchRepository(session_maker).init_search_index()
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error running migrations: {e}")
|
||||
raise
|
||||
|
||||
@@ -2,7 +2,7 @@ from httpx import ASGITransport, AsyncClient
|
||||
|
||||
from basic_memory.api.app import app as fastapi_app
|
||||
|
||||
BASE_URL = "http://test"
|
||||
BASE_URL = "memory://"
|
||||
|
||||
# Create shared async client
|
||||
client = AsyncClient(transport=ASGITransport(app=fastapi_app), base_url=BASE_URL)
|
||||
|
||||
@@ -13,6 +13,7 @@ from basic_memory.mcp.async_client import client
|
||||
from basic_memory.schemas import EntityResponse, DeleteEntitiesResponse
|
||||
from basic_memory.schemas.base import Entity
|
||||
from basic_memory.mcp.tools.utils import call_get, call_put, call_delete
|
||||
from basic_memory.schemas.memory import memory_url_path
|
||||
|
||||
|
||||
@mcp.tool(
|
||||
@@ -27,9 +28,29 @@ async def write_note(
|
||||
) -> EntityResponse | str:
|
||||
"""Write a markdown note to the knowledge base.
|
||||
|
||||
The content can include semantic observations and relations using markdown syntax.
|
||||
Relations can be specified either explicitly or through inline wiki-style links:
|
||||
|
||||
Observations format:
|
||||
`- [category] Observation text #tag1 #tag2 (optional context)`
|
||||
|
||||
Examples:
|
||||
`- [design] Files are the source of truth #architecture (All state comes from files)`
|
||||
`- [tech] Using SQLite for storage #implementation`
|
||||
`- [note] Need to add error handling #todo`
|
||||
|
||||
Relations format:
|
||||
- Explicit: `- relation_type [[Entity]] (optional context)`
|
||||
- Inline: Any `[[Entity]]` reference creates a relation
|
||||
|
||||
Examples:
|
||||
`- depends_on [[Content Parser]] (Need for semantic extraction)`
|
||||
`- implements [[Search Spec]] (Initial implementation)`
|
||||
`- This feature extends [[Base Design]] and uses [[Core Utils]]`
|
||||
|
||||
Args:
|
||||
title: The title of the note
|
||||
content: Markdown content for the note
|
||||
content: Markdown content for the note, can include observations and relations
|
||||
folder: the folder where the file should be saved
|
||||
tags: Optional list of tags to categorize the note
|
||||
verbose: If True, returns full EntityResponse with semantic info
|
||||
@@ -39,19 +60,32 @@ async def write_note(
|
||||
If verbose=True: EntityResponse with full semantic details
|
||||
|
||||
Examples:
|
||||
# Create a simple note
|
||||
# Note with both explicit and inline relations
|
||||
write_note(
|
||||
tile="Meeting Notes: Project Planning.md",
|
||||
content="# Key Points\\n\\n- Discussed timeline\\n- Set priorities"
|
||||
folder="notes"
|
||||
title="Search Implementation",
|
||||
content="# Search Component\\n\\n"
|
||||
"Implementation of the search feature, building on [[Core Search]].\\n\\n"
|
||||
"## Observations\\n"
|
||||
"- [tech] Using FTS5 for full-text search #implementation\\n"
|
||||
"- [design] Need pagination support #todo\\n\\n"
|
||||
"## Relations\\n"
|
||||
"- implements [[Search Spec]]\\n"
|
||||
"- depends_on [[Database Schema]]",
|
||||
folder="docs/components"
|
||||
)
|
||||
|
||||
# Create note with tags
|
||||
# Note with tags
|
||||
write_note(
|
||||
title="Security Review",
|
||||
content="# Findings\\n\\n1. Updated auth flow\\n2. Added rate limiting",
|
||||
folder="security",
|
||||
tags=["security", "development"]
|
||||
title="Error Handling Design",
|
||||
content="# Error Handling\\n\\n"
|
||||
"This design builds on [[Reliability Design]].\\n\\n"
|
||||
"## Approach\\n"
|
||||
"- [design] Use error codes #architecture\\n"
|
||||
"- [tech] Implement retry logic #implementation\\n\\n"
|
||||
"## Relations\\n"
|
||||
"- extends [[Base Error Handling]]",
|
||||
folder="docs/design",
|
||||
tags=["architecture", "reliability"]
|
||||
)
|
||||
"""
|
||||
logger.info(f"Writing note folder:'{folder}' title: '{title}'")
|
||||
@@ -75,28 +109,62 @@ async def write_note(
|
||||
return result if verbose else result.permalink
|
||||
|
||||
|
||||
@mcp.tool(description="Read a note's content by its title or permalink")
|
||||
@mcp.tool(description="Read note content by title, permalink, relation, or pattern")
|
||||
async def read_note(identifier: str) -> str:
|
||||
"""Get the markdown content of a note.
|
||||
Uses the resource router to return the actual file content.
|
||||
"""Get note content in unified diff format.
|
||||
|
||||
The content is returned in a unified diff inspired format:
|
||||
```
|
||||
--- memory://docs/example 2025-01-31T19:32:49 7d9f1c8b
|
||||
<document content>
|
||||
```
|
||||
|
||||
Multiple documents (from relations or pattern matches) are separated by
|
||||
additional headers.
|
||||
|
||||
Args:
|
||||
identifier: Note title or permalink
|
||||
identifier: Can be one of:
|
||||
- Note title ("Project Planning")
|
||||
- Note permalink ("docs/example")
|
||||
- Relation path ("docs/example/depends-on/other-doc")
|
||||
- Pattern match ("docs/*-architecture")
|
||||
|
||||
Returns:
|
||||
The note's markdown content
|
||||
Document content in unified diff format. For single documents, returns
|
||||
just that document's content. For relations or pattern matches, returns
|
||||
multiple documents separated by unified diff headers.
|
||||
|
||||
Examples:
|
||||
# Read by title
|
||||
read_note("Meeting Notes: Project Planning")
|
||||
# Single document
|
||||
content = await read_note("Project Planning")
|
||||
|
||||
# Read by permalink
|
||||
read_note("notes/project-planning")
|
||||
content = await read_note("docs/architecture/file-first")
|
||||
|
||||
Raises:
|
||||
ValueError: If the note cannot be found
|
||||
# Follow relation
|
||||
content = await read_note("docs/architecture/depends-on/docs/content-parser")
|
||||
|
||||
# Pattern matching
|
||||
content = await read_note("docs/*-architecture") # All architecture docs
|
||||
content = await read_note("docs/*/implements/*") # Find implementations
|
||||
|
||||
Output format:
|
||||
```
|
||||
--- memory://docs/example 2025-01-31T19:32:49 7d9f1c8b
|
||||
<first document content>
|
||||
|
||||
--- memory://docs/other 2025-01-30T15:45:22 a1b2c3d4
|
||||
<second document content>
|
||||
```
|
||||
|
||||
The headers include:
|
||||
- Full memory:// URI for the document
|
||||
- Last modified timestamp
|
||||
- Content checksum
|
||||
"""
|
||||
response = await call_get(client, f"/resource/{identifier}")
|
||||
logger.info(f"Reading note {identifier}")
|
||||
url = memory_url_path(identifier)
|
||||
response = await call_get(client, f"/resource/{url}")
|
||||
return response.text
|
||||
|
||||
|
||||
|
||||
@@ -68,24 +68,40 @@ class SearchRepository:
|
||||
|
||||
async def init_search_index(self):
|
||||
"""Create or recreate the search index."""
|
||||
|
||||
logger.info("Initializing search index")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await session.execute(CREATE_SEARCH_INDEX)
|
||||
await session.commit()
|
||||
try:
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
await session.execute(CREATE_SEARCH_INDEX)
|
||||
await session.commit()
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.error(f"Error initializing search index: {e}")
|
||||
raise e
|
||||
|
||||
def _quote_search_term(self, term: str) -> str:
|
||||
"""Add quotes if term contains special characters.
|
||||
For FTS5, special characters and phrases need to be quoted to be treated as a single token.
|
||||
def _prepare_search_term(self, term: str, is_prefix: bool = True) -> str:
|
||||
"""Prepare a search term for FTS5 query.
|
||||
|
||||
Args:
|
||||
term: The search term to prepare
|
||||
is_prefix: Whether to add prefix search capability (* suffix)
|
||||
|
||||
For FTS5:
|
||||
- Special characters and phrases need to be quoted
|
||||
- Terms with spaces or special chars need quotes
|
||||
"""
|
||||
# List of special characters that need quoting
|
||||
special_chars = ["/", "*", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
|
||||
if "*" in term:
|
||||
return term
|
||||
|
||||
# List of special characters that need quoting (excluding *)
|
||||
special_chars = ["/", "-", ".", " ", "(", ")", "[", "]", '"', "'"]
|
||||
|
||||
# Check if term contains any special characters
|
||||
if any(c in term for c in special_chars):
|
||||
# If the term already contains quotes, escape them
|
||||
needs_quotes = any(c in term for c in special_chars)
|
||||
|
||||
if needs_quotes:
|
||||
# If the term already contains quotes, escape them and add a wildcard
|
||||
term = term.replace('"', '""')
|
||||
return f'"{term}"'
|
||||
term = f'"{term}"*'
|
||||
|
||||
return term
|
||||
|
||||
async def search(
|
||||
@@ -106,14 +122,14 @@ class SearchRepository:
|
||||
|
||||
# Handle text search for title and content
|
||||
if search_text:
|
||||
search_text = self._quote_search_term(search_text.lower().strip())
|
||||
params["text"] = f"{search_text}*"
|
||||
search_text = self._prepare_search_term(search_text.strip())
|
||||
params["text"] = search_text
|
||||
conditions.append("(title MATCH :text OR content MATCH :text)")
|
||||
|
||||
# Handle title match search
|
||||
if title:
|
||||
title_text = self._quote_search_term(title.lower().strip())
|
||||
params["text"] = f"{title_text}*"
|
||||
title_text = self._prepare_search_term(title.strip())
|
||||
params["text"] = title_text
|
||||
conditions.append("title MATCH :text")
|
||||
|
||||
# Handle permalink exact search
|
||||
@@ -123,8 +139,15 @@ class SearchRepository:
|
||||
|
||||
# Handle permalink match search, supports *
|
||||
if permalink_match:
|
||||
params["permalink"] = self._quote_search_term(permalink_match)
|
||||
conditions.append("permalink MATCH :permalink")
|
||||
# Clean and prepare permalink for FTS5 GLOB match
|
||||
permalink_text = self._prepare_search_term(
|
||||
permalink_match.lower().strip(), is_prefix=False
|
||||
)
|
||||
params["permalink"] = permalink_text
|
||||
if "*" in permalink_match:
|
||||
conditions.append("permalink GLOB :permalink")
|
||||
else:
|
||||
conditions.append("permalink MATCH :permalink")
|
||||
|
||||
# Handle type filter
|
||||
if types:
|
||||
@@ -173,7 +196,7 @@ class SearchRepository:
|
||||
LIMIT :limit
|
||||
"""
|
||||
|
||||
# logger.debug(f"Search {sql} params: {params}")
|
||||
logger.debug(f"Search {sql} params: {params}")
|
||||
async with db.scoped_session(self.session_maker) as session:
|
||||
result = await session.execute(text(sql), params)
|
||||
rows = result.fetchall()
|
||||
@@ -199,8 +222,11 @@ class SearchRepository:
|
||||
for row in rows
|
||||
]
|
||||
|
||||
# for r in results:
|
||||
# logger.debug(f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}")
|
||||
logger.debug(f"Found {len(results)} search results")
|
||||
for r in results:
|
||||
logger.debug(
|
||||
f"Search result: type:{r.type} title: {r.title} permalink: {r.permalink} score: {r.score}"
|
||||
)
|
||||
|
||||
return results
|
||||
|
||||
@@ -233,7 +259,7 @@ class SearchRepository:
|
||||
"""),
|
||||
search_index_row.to_insert(),
|
||||
)
|
||||
logger.debug(f"indexed permalink {search_index_row.permalink}")
|
||||
logger.debug(f"indexed row {search_index_row}")
|
||||
await session.commit()
|
||||
|
||||
async def delete_by_permalink(self, permalink: str):
|
||||
|
||||
@@ -11,6 +11,7 @@ Key Features:
|
||||
4. Bulk operations return all affected items
|
||||
"""
|
||||
|
||||
from datetime import datetime
|
||||
from typing import List, Optional, Dict
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, AliasPath, AliasChoices
|
||||
@@ -43,6 +44,8 @@ class ObservationResponse(Observation, SQLAlchemyModel):
|
||||
}
|
||||
"""
|
||||
|
||||
permalink: Permalink
|
||||
|
||||
|
||||
class RelationResponse(Relation, SQLAlchemyModel):
|
||||
"""Response schema for relation operations.
|
||||
@@ -59,6 +62,8 @@ class RelationResponse(Relation, SQLAlchemyModel):
|
||||
}
|
||||
"""
|
||||
|
||||
permalink: Permalink
|
||||
|
||||
from_id: Permalink = Field(
|
||||
# use the permalink from the associated Entity
|
||||
# or the from_id value
|
||||
@@ -131,9 +136,12 @@ class EntityResponse(SQLAlchemyModel):
|
||||
file_path: str
|
||||
entity_type: EntityType
|
||||
entity_metadata: Optional[Dict] = None
|
||||
checksum: Optional[str] = None
|
||||
content_type: ContentType
|
||||
observations: List[ObservationResponse] = []
|
||||
relations: List[RelationResponse] = []
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class EntityListResponse(SQLAlchemyModel):
|
||||
|
||||
@@ -185,6 +185,11 @@ class EntityService(BaseService[EntityModel]):
|
||||
raise EntityNotFoundError(f"Entity not found: {permalink}")
|
||||
return db_entity
|
||||
|
||||
async def get_entities_by_id(self, ids: List[int]) -> Sequence[EntityModel]:
|
||||
"""Get specific entities and their relationships."""
|
||||
logger.debug(f"Getting entities: {ids}")
|
||||
return await self.repository.find_by_ids(ids)
|
||||
|
||||
async def get_entities_by_permalinks(self, permalinks: List[str]) -> Sequence[EntityModel]:
|
||||
"""Get specific nodes and their relationships."""
|
||||
logger.debug(f"Getting entities permalinks: {permalinks}")
|
||||
|
||||
@@ -16,9 +16,10 @@ class LinkResolver:
|
||||
|
||||
Uses a combination of exact matching and search-based resolution:
|
||||
1. Try exact permalink match (fastest)
|
||||
2. Try exact title match
|
||||
3. Fall back to search for fuzzy matching
|
||||
4. Generate new permalink if no match found
|
||||
2. Try permalink pattern match (for wildcards)
|
||||
3. Try exact title match
|
||||
4. Fall back to search for fuzzy matching
|
||||
5. Generate new permalink if no match found
|
||||
"""
|
||||
|
||||
def __init__(self, entity_repository: EntityRepository, search_service: SearchService):
|
||||
@@ -45,8 +46,8 @@ class LinkResolver:
|
||||
logger.debug(f"Found title match: {entity.title}")
|
||||
return entity
|
||||
|
||||
if use_search:
|
||||
# 3. Fall back to search for fuzzy matching on title if specified
|
||||
if use_search and "*" not in clean_text:
|
||||
# 3. Fall back to search for fuzzy matching on title
|
||||
results = await self.search_service.search(
|
||||
query=SearchQuery(title=clean_text, types=[SearchItemType.ENTITY]),
|
||||
)
|
||||
@@ -59,7 +60,7 @@ class LinkResolver:
|
||||
)
|
||||
return await self.entity_repository.get_by_permalink(best_match.permalink)
|
||||
|
||||
# if we couldn't find anything then return None
|
||||
# if we couldn't find anything then return None
|
||||
return None
|
||||
|
||||
def _normalize_link_text(self, link_text: str) -> Tuple[str, Optional[str]]:
|
||||
@@ -87,7 +88,7 @@ class LinkResolver:
|
||||
|
||||
return text, alias
|
||||
|
||||
def _select_best_match(self, search_text: str, results: List[SearchIndexRow]) -> Entity:
|
||||
def _select_best_match(self, search_text: str, results: List[SearchIndexRow]) -> SearchIndexRow:
|
||||
"""Select best match from search results.
|
||||
|
||||
Uses multiple criteria:
|
||||
|
||||
@@ -75,7 +75,7 @@ class SearchService:
|
||||
else None
|
||||
)
|
||||
|
||||
# permalink search
|
||||
# search
|
||||
results = await self.repository.search(
|
||||
search_text=query.text,
|
||||
permalink=query.permalink,
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
"""Types and utilities for file sync."""
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Set, Dict, Optional
|
||||
|
||||
from watchfiles import Change
|
||||
|
||||
from typing import Set, Dict
|
||||
|
||||
|
||||
@dataclass
|
||||
|
||||
@@ -1,458 +0,0 @@
|
||||
# Basic Memory Tasks
|
||||
|
||||
## Current Focus
|
||||
|
||||
### Observation Management
|
||||
|
||||
Implement update/remove functionality for observations with a focus on maintainability and consistency with our "
|
||||
filesystem is source of truth" principle.
|
||||
|
||||
Options under consideration:
|
||||
|
||||
1. Bulk Update Approach
|
||||
- Update all observations at once
|
||||
- Pros:
|
||||
- Simpler file operations
|
||||
- No need to match on observation content
|
||||
- Easier database synchronization
|
||||
- Very consistent with "filesystem is source of truth"
|
||||
- Cons:
|
||||
- Less efficient - rewrites everything for small changes
|
||||
- Potential concurrency implications
|
||||
|
||||
2. Tracked Observations Approach
|
||||
- Use markdown comments for observation IDs
|
||||
```markdown
|
||||
# Entity Name
|
||||
type: entity_type
|
||||
|
||||
## Observations
|
||||
- <!-- obs-id: abc123 -->
|
||||
This is an observation
|
||||
```
|
||||
- Pros:
|
||||
- Can track individual observations
|
||||
- Enables precise updates/deletes
|
||||
- Cons:
|
||||
- More complex markdown parsing
|
||||
- IDs visible in markdown
|
||||
|
||||
3. Diff-based Approach
|
||||
- Implement observation-aware diffing
|
||||
- Track changes at observation level
|
||||
- Pros:
|
||||
- More efficient updates
|
||||
- Preserves manual edits
|
||||
- Cons:
|
||||
- More complex implementation
|
||||
- Need to handle merge conflicts
|
||||
|
||||
4. Position-based Management
|
||||
- Track observations by their position/order
|
||||
- Pros:
|
||||
- No need for explicit IDs
|
||||
- Clean markdown
|
||||
- Cons:
|
||||
- Fragile if order changes
|
||||
- Hard to handle concurrent edits
|
||||
|
||||
## Completed
|
||||
|
||||
- [x] Extract file operations to fileio.py module
|
||||
- [x] Update EntityService to use fileio functions
|
||||
- [x] Initial ObservationService implementation
|
||||
- [x] Basic test coverage
|
||||
|
||||
## Future Work
|
||||
|
||||
- [ ] Implement observation updates/removals (exploring options above)
|
||||
- [ ] Proper session management for concurrent operations
|
||||
- [ ] EntityService tests using new fileio module
|
||||
- [ ] More sophisticated search functionality
|
||||
- [ ] Handle markdown formatting edge cases
|
||||
|
||||
## TODO
|
||||
|
||||
### refactor input schema
|
||||
|
||||
1. Observations Format:
|
||||
Old (JSON) way I tried first:
|
||||
|
||||
```python
|
||||
"observations": ["First observation", "Second observation"] # Simple string array
|
||||
```
|
||||
|
||||
New required format:
|
||||
|
||||
```python
|
||||
"observations": [
|
||||
{"content": "First observation"},
|
||||
{"content": "Second observation"}
|
||||
] # Array of objects with content field
|
||||
```
|
||||
|
||||
2. Relations Format:
|
||||
Old way:
|
||||
|
||||
```python
|
||||
"relations": [
|
||||
{"from": "EntityName", "to": "OtherEntity", "relationType": "relates_to"} # Using names
|
||||
]
|
||||
```
|
||||
|
||||
New format:
|
||||
|
||||
```python
|
||||
"relations": [
|
||||
{"fromId": "20241210-entity-id", "toId": "20241210-other-id", "relationType": "relates_to"} # Using IDs
|
||||
]
|
||||
```
|
||||
|
||||
My preferences:
|
||||
|
||||
1. For observations: The simple string array felt more intuitive for basic use, but I can see the benefits of the object
|
||||
format:
|
||||
- Allows for additional metadata (context, timestamps, etc.)
|
||||
- More explicit about what each field means
|
||||
- Consistent with how we'd want to store this in a database
|
||||
|
||||
2. For relations: Using IDs is technically better but requires an extra lookup step in my workflow:
|
||||
- I have to first create the entities to get their IDs
|
||||
- Then use those IDs to create relations
|
||||
- Makes it harder to create entities and relations in a single step
|
||||
|
||||
Suggestions for making it more intuitive:
|
||||
|
||||
1. For observations: We could have a helper function that accepts either format:
|
||||
|
||||
```python
|
||||
# Both would work:
|
||||
create_entities([{
|
||||
"name": "Entity",
|
||||
"entityType": "type",
|
||||
"observations": ["Simple string obs"] # Auto-converted to object format
|
||||
}])
|
||||
|
||||
create_entities([{
|
||||
"name": "Entity",
|
||||
"entityType": "type",
|
||||
"observations": [{"content": "Full object obs"}] # Native format
|
||||
}])
|
||||
```
|
||||
|
||||
2. For relations: Maybe allow a name-based helper function:
|
||||
|
||||
```python
|
||||
# Instead of requiring IDs:
|
||||
create_relations_by_name([{
|
||||
"from": "EntityName",
|
||||
"to": "OtherEntity",
|
||||
"relationType": "relates_to"
|
||||
}])
|
||||
```
|
||||
|
||||
3. A combined creation function for when we want to create entities and their relations together:
|
||||
|
||||
```python
|
||||
create_entity_with_relations({
|
||||
"entity": {
|
||||
"name": "NewEntity",
|
||||
"entityType": "type",
|
||||
"observations": ["Obs 1", "Obs 2"]
|
||||
},
|
||||
"relations": [{
|
||||
"to": "ExistingEntity",
|
||||
"relationType": "relates_to"
|
||||
}]
|
||||
})
|
||||
```
|
||||
|
||||
The current schema is more "correct" from a software engineering perspective, but these helpers could make it more
|
||||
natural to use while maintaining the rigorous underlying structure. What do you think about these suggestions? Would you
|
||||
prefer to keep it strict for clarity, or add some convenience layers?
|
||||
|
||||
### entity file organization
|
||||
|
||||
- my problems
|
||||
- wall of text is hard to scan visually
|
||||
- files are not ordered or grouped granular enough. Only `<date>_<entity_name>.md`
|
||||
- hard to tell when a new entity file is added
|
||||
|
||||
Possible fixes:
|
||||
|
||||
- use subdirectories?
|
||||
|
||||
## TASKS
|
||||
|
||||
1. **Core Functionality Improvements**
|
||||
- [x] entity.description addition
|
||||
- [x] subdirectories
|
||||
- Paul needs this for markdown view
|
||||
- [x] refine directory sprawl
|
||||
- [ ] improve tool api
|
||||
- [ ] Choose and implement observation update/removal strategy
|
||||
- [ ] Handle relationship updates in files
|
||||
- [ ] Complete full CRUD operations
|
||||
- delete
|
||||
- [ ] Improve search functionality (currently broken as we discovered)
|
||||
|
||||
### Suggested Sequence
|
||||
|
||||
1. **Schema Update First**
|
||||
- Add `entity.description` field
|
||||
- rename entity.references?
|
||||
- This affects database, Pydantic models, and file format
|
||||
- Good foundation for other changes
|
||||
|
||||
2. **File Organization**
|
||||
- Add subdirectory support
|
||||
- Affects:
|
||||
- File path handling
|
||||
- Entity loading/saving
|
||||
- URI resolution
|
||||
- Will make Paul's markdown viewing experience better
|
||||
|
||||
3. **Tool API Improvements**
|
||||
- Cleaner input/output schemas
|
||||
- More consistent patterns
|
||||
- Better error handling
|
||||
- This sets us up for implementing the remaining operations
|
||||
|
||||
4. **Core Operations**
|
||||
- Implement delete operations
|
||||
- Update/remove observations
|
||||
- Relationship updates in files
|
||||
- Building on the improved API
|
||||
|
||||
5. **Search Fix**
|
||||
- Can properly tackle this after file organization
|
||||
- Will benefit from improved schema
|
||||
|
||||
Would you like me to:
|
||||
1. Start with the schema update for entity.description?
|
||||
2. Plan out the subdirectory implementation?
|
||||
3. Or focus on a different area?
|
||||
|
||||
I think the schema update would be a clean, contained change to start with, but I'm happy to tackle whichever part you think would be most valuable first.
|
||||
|
||||
|
||||
2. **Robustness & Testing**
|
||||
- Fix DI issues
|
||||
- Learn from fastmcp patterns
|
||||
- Markdown service
|
||||
- markdown.py
|
||||
-python-frontmatter
|
||||
- Complete test coverage
|
||||
- Expand testing across services
|
||||
- 100% coverate
|
||||
- Improve error handling and logging
|
||||
- Add comprehensive type hints
|
||||
|
||||
3. **Architecture Improvements**
|
||||
- Handle concurrent file operations safely
|
||||
- Implement proper session management
|
||||
- Balance file operations and DB sync
|
||||
- Handle markdown formatting edge cases
|
||||
|
||||
4. **Documentation & Infrastructure**
|
||||
- Document core modules
|
||||
- Implement proper backup strategy
|
||||
- Add file change versioning
|
||||
- Improve CLI interface
|
||||
|
||||
|
||||
## Ideas
|
||||
|
||||
- need update tool
|
||||
|
||||
### 2-way sync
|
||||
|
||||
- Enable updates to the markdown files to be able to be seen by AI
|
||||
- possible via tool sync
|
||||
- filesystem notifications via agent?
|
||||
- Claude can use `file_write` tool to edit Entity files also
|
||||
|
||||
### Projects
|
||||
|
||||
- support multiple projects
|
||||
- figure out flow
|
||||
- load project at startup?
|
||||
- switch project during chat?
|
||||
|
||||
### References
|
||||
|
||||
- better support for ref:// references
|
||||
- should we call them memory://<project>/<entity>
|
||||
- use Prompt to invoke reference uri
|
||||
|
||||
### Entity File format
|
||||
|
||||
- Consider using wiki link format:
|
||||
[[20241210-ai_human_collaboration_model]] improves
|
||||
|
||||
instead of
|
||||
[20241210-basic-memory] validates_design_of
|
||||
|
||||
- use frontmatter for file header
|
||||
|
||||
### Code
|
||||
|
||||
- cleanup server.py
|
||||
- use markdown.py?
|
||||
- fastmcp?
|
||||
- Entity.description
|
||||
|
||||
### Other Features
|
||||
|
||||
- Search
|
||||
- webui
|
||||
- python notebook?
|
||||
- vector eb extension for sqlite
|
||||
|
||||
#### Obsidian
|
||||
|
||||
- use as file viewer editor
|
||||
- create knowledge graph?
|
||||
|
||||
#### Save chats
|
||||
|
||||
I'll sketch out a potential schema for chat sessions in our system:
|
||||
|
||||
```
|
||||
{
|
||||
`entities`: [
|
||||
{
|
||||
`name`: `Chat_Entity_Design`,
|
||||
`entityType`: `design_proposal`,
|
||||
`observations`: [
|
||||
{
|
||||
`content`: `Entity Schema:
|
||||
type: chat_session
|
||||
date: ISO timestamp
|
||||
participants: [human, ai]
|
||||
topic: string
|
||||
context_refs: [memory://<project>/<entity_id>]
|
||||
key_decisions: [string]
|
||||
summary: string`
|
||||
},
|
||||
{
|
||||
`content`: `File Format (frontmatter):
|
||||
---
|
||||
type: chat_session
|
||||
date: 2024-12-10T10:30:00Z
|
||||
participants:
|
||||
- human: paul
|
||||
- ai: claude
|
||||
topic: Chat Session Storage Design
|
||||
context_refs:
|
||||
- memory://basic-memory/20241210-chat-entity-design
|
||||
key_decisions:
|
||||
- Store chats as first-class entities
|
||||
- Use memory:// URI scheme
|
||||
summary: Designed schema for chat storage
|
||||
---`
|
||||
},
|
||||
{
|
||||
`content`: `Example Relations:
|
||||
- chat_session references design_proposal
|
||||
- chat_session implements feature
|
||||
- chat_session discusses concept
|
||||
- chat_session continues_from previous_chat`
|
||||
},
|
||||
{
|
||||
`content`: `Auto-generation approach:
|
||||
1. At end of chat, Claude summarizes key points
|
||||
2. Creates chat entity with metadata
|
||||
3. Adds relations to referenced entities
|
||||
4. Updates any entities modified during chat`
|
||||
},
|
||||
{
|
||||
`content`: `Integration points:
|
||||
- memory:// URI scheme for references
|
||||
- Relations to track context chain
|
||||
- File format matches entity schema
|
||||
- Automatic metadata generation`
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
```
|
||||
{
|
||||
`entities`: [
|
||||
{
|
||||
`name`: `Chat_Session_Workflow`,
|
||||
`entityType`: `process`,
|
||||
`observations`: [
|
||||
{
|
||||
`content`: `Start of chat:
|
||||
- Load any referenced context via memory:// URIs
|
||||
- Initialize new chat session metadata
|
||||
- Track entities being discussed/modified`
|
||||
},
|
||||
{
|
||||
`content`: `During chat:
|
||||
- Update references as new entities are mentioned
|
||||
- Track key decisions and changes
|
||||
- Note any new relations discovered`
|
||||
},
|
||||
{
|
||||
`content`: `End of chat:
|
||||
- Generate summary and key points
|
||||
- Create chat session entity
|
||||
- Add relations to referenced entities
|
||||
- Update any modified entities`
|
||||
},
|
||||
{
|
||||
`content`: `Future chat reference:
|
||||
- Can reference previous chat via memory:// URI
|
||||
- Loads relevant context and decisions
|
||||
- Creates continues_from relation automatically`
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Key aspects of this design:
|
||||
|
||||
1. Chat Session as Entity
|
||||
|
||||
- Clear metadata (participants, date, topic)
|
||||
- Explicit references to other entities via memory:// URIs
|
||||
- Captures key decisions and summary
|
||||
- Uses frontmatter for clean file format
|
||||
|
||||
2. Rich Relations
|
||||
|
||||
- Can track chat-to-chat continuity
|
||||
- Links to entities discussed/modified
|
||||
- Captures purpose (discusses, implements, etc.)
|
||||
- Maintains context chain
|
||||
|
||||
3. Automated Workflow
|
||||
|
||||
- I could generate metadata during chat
|
||||
- Create summary and relations at end
|
||||
- Update any modified entities
|
||||
- Makes context loading explicit
|
||||
|
||||
4. Integration
|
||||
|
||||
- memory:// URI scheme fits naturally
|
||||
- File format matches our entity pattern
|
||||
- Two-way links between chats and entities
|
||||
- Clear context persistence model
|
||||
|
||||
The nice thing about this design is that it makes chat sessions "first-class citizens" in our knowledge graph while
|
||||
keeping everything in human-readable markdown files.
|
||||
|
||||
Would you like me to:
|
||||
|
||||
1. Sketch out what this chat would look like as an entity?
|
||||
2. Detail the MCP tool changes needed?
|
||||
3. Mock up the file format more precisely?
|
||||
4. Something else?
|
||||
@@ -5,6 +5,8 @@ from datetime import datetime, timezone
|
||||
import pytest
|
||||
from pathlib import Path
|
||||
|
||||
from basic_memory.schemas import EntityResponse
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_content(client, test_config, entity_repository):
|
||||
@@ -67,7 +69,7 @@ async def test_get_resource_missing_entity(client):
|
||||
"""Test 404 when entity doesn't exist."""
|
||||
response = await client.get("/resource/does/not/exist")
|
||||
assert response.status_code == 404
|
||||
assert "Entity not found" in response.json()["detail"]
|
||||
assert "Resource not found" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -89,3 +91,139 @@ async def test_get_resource_missing_file(client, test_config, entity_repository)
|
||||
response = await client.get(f"/resource/{entity.permalink}")
|
||||
assert response.status_code == 404
|
||||
assert "File not found" in response.json()["detail"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_observation(client, test_config, entity_repository):
|
||||
"""Test getting content by observation permalink."""
|
||||
# Create entity
|
||||
content = "# Test Content\n\n- [note] an observation."
|
||||
data = {
|
||||
"title": "Test Entity",
|
||||
"folder": "test",
|
||||
"entity_type": "test",
|
||||
"content": f"{content}",
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
entity = EntityResponse(**entity_response)
|
||||
|
||||
assert len(entity.observations) == 1
|
||||
observation = entity.observations[0]
|
||||
|
||||
# Test getting the content via the observation
|
||||
response = await client.get(f"/resource/{observation.permalink}")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == "text/markdown; charset=utf-8"
|
||||
assert (
|
||||
"""
|
||||
---
|
||||
title: Test Entity
|
||||
type: test
|
||||
permalink: test/test-entity
|
||||
---
|
||||
|
||||
# Test Content
|
||||
|
||||
- [note] an observation.
|
||||
""".strip()
|
||||
in response.text
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_entities(client, test_config, entity_repository):
|
||||
"""Test getting content by permalink match."""
|
||||
# Create entity
|
||||
content1 = "# Test Content\n"
|
||||
data = {
|
||||
"title": "Test Entity",
|
||||
"folder": "test",
|
||||
"entity_type": "test",
|
||||
"content": f"{content1}",
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
entity1 = EntityResponse(**entity_response)
|
||||
|
||||
content2 = "# Related Content\n- links to [[Test Entity]]"
|
||||
data = {
|
||||
"title": "Related Entity",
|
||||
"folder": "test",
|
||||
"entity_type": "test",
|
||||
"content": f"{content2}",
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
entity2 = EntityResponse(**entity_response)
|
||||
|
||||
assert len(entity2.relations) == 1
|
||||
relation = entity2.relations[0]
|
||||
|
||||
# Test getting the content via the relation
|
||||
response = await client.get("/resource/test/*")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == "text/markdown; charset=utf-8"
|
||||
assert (
|
||||
f"""
|
||||
--- memory://test/test-entity {entity1.updated_at.isoformat()} {entity1.checksum[:8]}
|
||||
|
||||
# Test Content
|
||||
|
||||
--- memory://test/related-entity {entity2.updated_at.isoformat()} {entity2.checksum[:8]}
|
||||
|
||||
# Related Content
|
||||
- links to [[Test Entity]]
|
||||
|
||||
""".strip()
|
||||
in response.text
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_resource_relation(client, test_config, entity_repository):
|
||||
"""Test getting content by relation permalink."""
|
||||
# Create entity
|
||||
content1 = "# Test Content\n"
|
||||
data = {
|
||||
"title": "Test Entity",
|
||||
"folder": "test",
|
||||
"entity_type": "test",
|
||||
"content": f"{content1}",
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
entity1 = EntityResponse(**entity_response)
|
||||
|
||||
content2 = "# Related Content\n- links to [[Test Entity]]"
|
||||
data = {
|
||||
"title": "Related Entity",
|
||||
"folder": "test",
|
||||
"entity_type": "test",
|
||||
"content": f"{content2}",
|
||||
}
|
||||
response = await client.post("/knowledge/entities", json=data)
|
||||
entity_response = response.json()
|
||||
entity2 = EntityResponse(**entity_response)
|
||||
|
||||
assert len(entity2.relations) == 1
|
||||
relation = entity2.relations[0]
|
||||
|
||||
# Test getting the content via the relation
|
||||
response = await client.get(f"/resource/{relation.permalink}")
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == "text/markdown; charset=utf-8"
|
||||
assert (
|
||||
f"""
|
||||
--- memory://test/test-entity {entity1.updated_at.isoformat()} {entity1.checksum[:8]}
|
||||
|
||||
# Test Content
|
||||
|
||||
--- memory://test/related-entity {entity2.updated_at.isoformat()} {entity2.checksum[:8]}
|
||||
|
||||
# Related Content
|
||||
- links to [[Test Entity]]
|
||||
|
||||
""".strip()
|
||||
in response.text
|
||||
)
|
||||
|
||||
@@ -0,0 +1,276 @@
|
||||
"""Tests for import_chatgpt command."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import import_app, app
|
||||
from basic_memory.cli.commands import import_chatgpt
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation():
|
||||
"""Sample ChatGPT conversation data for testing."""
|
||||
return {
|
||||
"title": "Test Conversation",
|
||||
"create_time": 1736616594.24054, # Example timestamp
|
||||
"update_time": 1736616603.164995,
|
||||
"mapping": {
|
||||
"root": {"id": "root", "message": None, "parent": None, "children": ["msg1"]},
|
||||
"msg1": {
|
||||
"id": "msg1",
|
||||
"message": {
|
||||
"id": "msg1",
|
||||
"author": {"role": "user", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {"content_type": "text", "parts": ["Hello, this is a test message"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": ["msg2"],
|
||||
},
|
||||
"msg2": {
|
||||
"id": "msg2",
|
||||
"message": {
|
||||
"id": "msg2",
|
||||
"author": {"role": "assistant", "name": None, "metadata": {}},
|
||||
"create_time": 1736616603.164995,
|
||||
"content": {"content_type": "text", "parts": ["This is a test response"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "msg1",
|
||||
"children": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation_with_code():
|
||||
"""Sample conversation with code block."""
|
||||
conversation = {
|
||||
"title": "Code Test",
|
||||
"create_time": 1736616594.24054,
|
||||
"update_time": 1736616603.164995,
|
||||
"mapping": {
|
||||
"root": {"id": "root", "message": None, "parent": None, "children": ["msg1"]},
|
||||
"msg1": {
|
||||
"id": "msg1",
|
||||
"message": {
|
||||
"id": "msg1",
|
||||
"author": {"role": "assistant", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {
|
||||
"content_type": "code",
|
||||
"language": "python",
|
||||
"text": "def hello():\n print('Hello world!')",
|
||||
},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
"msg2": {
|
||||
"id": "msg2",
|
||||
"message": {
|
||||
"id": "msg2",
|
||||
"author": {"role": "assistant", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
return conversation
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation_with_hidden():
|
||||
"""Sample conversation with hidden messages."""
|
||||
conversation = {
|
||||
"title": "Hidden Test",
|
||||
"create_time": 1736616594.24054,
|
||||
"update_time": 1736616603.164995,
|
||||
"mapping": {
|
||||
"root": {
|
||||
"id": "root",
|
||||
"message": None,
|
||||
"parent": None,
|
||||
"children": ["visible", "hidden"],
|
||||
},
|
||||
"visible": {
|
||||
"id": "visible",
|
||||
"message": {
|
||||
"id": "visible",
|
||||
"author": {"role": "user", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {"content_type": "text", "parts": ["Visible message"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
"hidden": {
|
||||
"id": "hidden",
|
||||
"message": {
|
||||
"id": "hidden",
|
||||
"author": {"role": "system", "name": None, "metadata": {}},
|
||||
"create_time": 1736616594.24054,
|
||||
"content": {"content_type": "text", "parts": ["Hidden message"]},
|
||||
"status": "finished_successfully",
|
||||
"metadata": {"is_visually_hidden_from_conversation": True},
|
||||
},
|
||||
"parent": "root",
|
||||
"children": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
return conversation
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_chatgpt_json(tmp_path, sample_conversation):
|
||||
"""Create a sample ChatGPT JSON file."""
|
||||
json_file = tmp_path / "conversations.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation], f)
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_chatgpt_json(tmp_path, sample_chatgpt_json):
|
||||
"""Test importing conversations from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_chatgpt.process_chatgpt_json(sample_chatgpt_json, tmp_path, processor)
|
||||
|
||||
assert results["conversations"] == 1
|
||||
assert results["messages"] == 2
|
||||
|
||||
# Check conversation file exists
|
||||
conv_path = tmp_path / "20250111-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
|
||||
# Check content formatting
|
||||
content = conv_path.read_text()
|
||||
assert "# Test Conversation" in content
|
||||
assert "### User" in content
|
||||
assert "Hello, this is a test message" in content
|
||||
assert "### Assistant" in content
|
||||
assert "This is a test response" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_code_blocks(tmp_path, sample_conversation_with_code):
|
||||
"""Test handling of code blocks."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
# Create test file
|
||||
json_file = tmp_path / "code_test.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation_with_code], f)
|
||||
|
||||
await import_chatgpt.process_chatgpt_json(json_file, tmp_path, processor)
|
||||
|
||||
# Check content
|
||||
conv_path = tmp_path / "20250111-code-test.md"
|
||||
content = conv_path.read_text()
|
||||
assert "```python" in content
|
||||
assert "def hello():" in content
|
||||
assert "```" in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hidden_messages(tmp_path, sample_conversation_with_hidden):
|
||||
"""Test handling of hidden messages."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
# Create test file
|
||||
json_file = tmp_path / "hidden_test.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation_with_hidden], f)
|
||||
|
||||
results = await import_chatgpt.process_chatgpt_json(json_file, tmp_path, processor)
|
||||
|
||||
# Should only count visible messages
|
||||
assert results["messages"] == 1
|
||||
|
||||
# Check content
|
||||
conv_path = tmp_path / "20250111-hidden-test.md"
|
||||
content = conv_path.read_text()
|
||||
assert "Visible message" in content
|
||||
assert "Hidden message" not in content
|
||||
|
||||
|
||||
def test_import_chatgpt_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import", "chatgpt", "--conversations-json", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_chatgpt_command_success(tmp_path, sample_chatgpt_json, monkeypatch):
|
||||
"""Test successful conversation import via command."""
|
||||
# Set up test environment
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
import_app, ["chatgpt", "--conversations-json", str(sample_chatgpt_json)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Imported 1 conversations" in result.output
|
||||
assert "Containing 2 messages" in result.output
|
||||
|
||||
|
||||
def test_import_chatgpt_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(import_app, ["chatgpt", "--conversations-json", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_chatgpt_with_custom_folder(tmp_path, sample_chatgpt_json, monkeypatch):
|
||||
"""Test import with custom conversations folder."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
conversations_folder = "chats"
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"import",
|
||||
"chatgpt",
|
||||
"--conversations-json",
|
||||
str(sample_chatgpt_json),
|
||||
"--folder",
|
||||
conversations_folder,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check files in custom folder
|
||||
conv_path = tmp_path / conversations_folder / "20250111-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Tests for import_claude command (chat conversations)."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands import import_claude_conversations as import_claude
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversation():
|
||||
"""Sample conversation data for testing."""
|
||||
return {
|
||||
"uuid": "test-uuid",
|
||||
"name": "Test Conversation",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"chat_messages": [
|
||||
{
|
||||
"uuid": "msg-1",
|
||||
"text": "Hello, this is a test",
|
||||
"sender": "human",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"content": [{"type": "text", "text": "Hello, this is a test"}],
|
||||
},
|
||||
{
|
||||
"uuid": "msg-2",
|
||||
"text": "Response to test",
|
||||
"sender": "assistant",
|
||||
"created_at": "2025-01-05T20:55:40.123456+00:00",
|
||||
"content": [{"type": "text", "text": "Response to test"}],
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_conversations_json(tmp_path, sample_conversation):
|
||||
"""Create a sample conversations.json file."""
|
||||
json_file = tmp_path / "conversations.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_conversation], f)
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_chat_json(tmp_path, sample_conversations_json):
|
||||
"""Test importing conversations from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_claude.process_conversations_json(
|
||||
sample_conversations_json, tmp_path, processor
|
||||
)
|
||||
|
||||
assert results["conversations"] == 1
|
||||
assert results["messages"] == 2
|
||||
|
||||
# Check conversation file
|
||||
conv_path = tmp_path / "20250105-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
content = conv_path.read_text()
|
||||
|
||||
# Check content formatting
|
||||
assert "### Human" in content
|
||||
assert "Hello, this is a test" in content
|
||||
assert "### Assistant" in content
|
||||
assert "Response to test" in content
|
||||
|
||||
|
||||
def test_import_conversations_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import", "claude", "conversations", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_conversations_command_success(tmp_path, sample_conversations_json, monkeypatch):
|
||||
"""Test successful conversation import via command."""
|
||||
# Set up test environment
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app, ["import", "claude", "conversations", str(sample_conversations_json)]
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Imported 1 conversations" in result.output
|
||||
assert "Containing 2 messages" in result.output
|
||||
|
||||
|
||||
def test_import_conversations_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(app, ["import", "claude", "conversations", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_conversations_with_custom_folder(tmp_path, sample_conversations_json, monkeypatch):
|
||||
"""Test import with custom conversations folder."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
conversations_folder = "chats"
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"import",
|
||||
"claude",
|
||||
"conversations",
|
||||
str(sample_conversations_json),
|
||||
"--folder",
|
||||
conversations_folder,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check files in custom folder
|
||||
conv_path = tmp_path / conversations_folder / "20250105-test-conversation.md"
|
||||
assert conv_path.exists()
|
||||
|
||||
|
||||
def test_import_conversation_with_attachments(tmp_path):
|
||||
"""Test importing conversation with attachments."""
|
||||
# Create conversation with attachments
|
||||
conversation = {
|
||||
"uuid": "test-uuid",
|
||||
"name": "Test With Attachments",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"chat_messages": [
|
||||
{
|
||||
"uuid": "msg-1",
|
||||
"text": "Here's a file",
|
||||
"sender": "human",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"content": [{"type": "text", "text": "Here's a file"}],
|
||||
"attachments": [
|
||||
{"file_name": "test.txt", "extracted_content": "Test file content"}
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
json_file = tmp_path / "with_attachments.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([conversation], f)
|
||||
|
||||
# Set up environment
|
||||
config.home = tmp_path
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import", "claude", "conversations", str(json_file)])
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check attachment formatting
|
||||
conv_path = tmp_path / "conversations/20250105-test-with-attachments.md"
|
||||
content = conv_path.read_text()
|
||||
assert "**Attachment: test.txt**" in content
|
||||
assert "```" in content
|
||||
assert "Test file content" in content
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Tests for import_claude_projects command."""
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.commands import import_claude_projects
|
||||
from basic_memory.config import config
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
# Set up CLI runner
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_project():
|
||||
"""Sample project data for testing."""
|
||||
return {
|
||||
"uuid": "test-uuid",
|
||||
"name": "Test Project",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"prompt_template": "# Test Prompt\n\nThis is a test prompt.",
|
||||
"docs": [
|
||||
{
|
||||
"uuid": "doc-uuid-1",
|
||||
"filename": "Test Document",
|
||||
"content": "# Test Document\n\nThis is test content.",
|
||||
"created_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
},
|
||||
{
|
||||
"uuid": "doc-uuid-2",
|
||||
"filename": "Another Document",
|
||||
"content": "# Another Document\n\nMore test content.",
|
||||
"created_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_projects_json(tmp_path, sample_project):
|
||||
"""Create a sample projects.json file."""
|
||||
json_file = tmp_path / "projects.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([sample_project], f)
|
||||
return json_file
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_projects_json(tmp_path, sample_projects_json):
|
||||
"""Test importing projects from JSON."""
|
||||
entity_parser = EntityParser(tmp_path)
|
||||
processor = MarkdownProcessor(entity_parser)
|
||||
|
||||
results = await import_claude_projects.process_projects_json(
|
||||
sample_projects_json, tmp_path, processor
|
||||
)
|
||||
|
||||
assert results["documents"] == 2
|
||||
assert results["prompts"] == 1
|
||||
|
||||
# Check project directory structure
|
||||
project_dir = tmp_path / "test-project"
|
||||
assert project_dir.exists()
|
||||
assert (project_dir / "docs").exists()
|
||||
assert (project_dir / "prompt-template.md").exists()
|
||||
|
||||
# Check document files
|
||||
doc1 = project_dir / "docs/test-document.md"
|
||||
assert doc1.exists()
|
||||
content1 = doc1.read_text()
|
||||
assert "# Test Document" in content1
|
||||
assert "This is test content" in content1
|
||||
|
||||
# Check prompt template
|
||||
prompt = project_dir / "prompt-template.md"
|
||||
assert prompt.exists()
|
||||
prompt_content = prompt.read_text()
|
||||
assert "# Test Prompt" in prompt_content
|
||||
assert "This is a test prompt" in prompt_content
|
||||
|
||||
|
||||
def test_import_projects_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
|
||||
def test_import_projects_command_success(tmp_path, sample_projects_json, monkeypatch):
|
||||
"""Test successful project import via command."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(sample_projects_json)])
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Imported 2 project documents" in result.output
|
||||
assert "Imported 1 prompt templates" in result.output
|
||||
|
||||
|
||||
def test_import_projects_command_invalid_json(tmp_path):
|
||||
"""Test error handling for invalid JSON."""
|
||||
# Create invalid JSON file
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
|
||||
def test_import_projects_with_base_folder(tmp_path, sample_projects_json, monkeypatch):
|
||||
"""Test import with custom base folder."""
|
||||
# Set up test environment
|
||||
config.home = tmp_path
|
||||
base_folder = "claude-exports"
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"import",
|
||||
"claude",
|
||||
"projects",
|
||||
str(sample_projects_json),
|
||||
"--base-folder",
|
||||
base_folder,
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 0
|
||||
|
||||
# Check files in base folder
|
||||
project_dir = tmp_path / base_folder / "test-project"
|
||||
assert project_dir.exists()
|
||||
assert (project_dir / "docs").exists()
|
||||
assert (project_dir / "prompt-template.md").exists()
|
||||
|
||||
|
||||
def test_import_project_without_prompt(tmp_path):
|
||||
"""Test importing project without prompt template."""
|
||||
# Create project without prompt
|
||||
project = {
|
||||
"uuid": "test-uuid",
|
||||
"name": "No Prompt Project",
|
||||
"created_at": "2025-01-05T20:55:32.499880+00:00",
|
||||
"updated_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
"docs": [
|
||||
{
|
||||
"uuid": "doc-uuid-1",
|
||||
"filename": "Test Document",
|
||||
"content": "# Test Document\n\nContent.",
|
||||
"created_at": "2025-01-05T20:56:39.477600+00:00",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
json_file = tmp_path / "no_prompt.json"
|
||||
with open(json_file, "w") as f:
|
||||
json.dump([project], f)
|
||||
|
||||
# Set up environment
|
||||
config.home = tmp_path
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import", "claude", "projects", str(json_file)])
|
||||
assert result.exit_code == 0
|
||||
assert "Imported 1 project documents" in result.output
|
||||
assert "Imported 0 prompt templates" in result.output
|
||||
@@ -4,7 +4,7 @@ import json
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from basic_memory.cli.app import app
|
||||
from basic_memory.cli.app import import_app
|
||||
from basic_memory.cli.commands import import_memory_json
|
||||
from basic_memory.markdown import EntityParser, MarkdownProcessor
|
||||
|
||||
@@ -72,7 +72,7 @@ async def test_get_markdown_processor(tmp_path, monkeypatch):
|
||||
def test_import_json_command_file_not_found(tmp_path):
|
||||
"""Test error handling for nonexistent file."""
|
||||
nonexistent = tmp_path / "nonexistent.json"
|
||||
result = runner.invoke(app, ["import-json", str(nonexistent)])
|
||||
result = runner.invoke(import_app, ["memory-json", str(nonexistent)])
|
||||
assert result.exit_code == 1
|
||||
assert "File not found" in result.output
|
||||
|
||||
@@ -83,7 +83,7 @@ def test_import_json_command_success(tmp_path, sample_json_file, monkeypatch):
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import-json", str(sample_json_file)])
|
||||
result = runner.invoke(import_app, ["memory-json", str(sample_json_file)])
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
assert "Created 1 entities" in result.output
|
||||
@@ -96,7 +96,7 @@ def test_import_json_command_invalid_json(tmp_path):
|
||||
invalid_file = tmp_path / "invalid.json"
|
||||
invalid_file.write_text("not json")
|
||||
|
||||
result = runner.invoke(app, ["import-json", str(invalid_file)])
|
||||
result = runner.invoke(import_app, ["memory-json", str(invalid_file)])
|
||||
assert result.exit_code == 1
|
||||
assert "Error during import" in result.output
|
||||
|
||||
@@ -129,6 +129,6 @@ def test_import_json_command_handle_old_format(tmp_path):
|
||||
monkeypatch.setenv("HOME", str(tmp_path))
|
||||
|
||||
# Run import
|
||||
result = runner.invoke(app, ["import-json", str(json_file)])
|
||||
result = runner.invoke(import_app, ["memory-json", str(json_file)])
|
||||
assert result.exit_code == 0
|
||||
assert "Import complete" in result.output
|
||||
|
||||
+2
-26
@@ -7,7 +7,6 @@ from datetime import datetime, timezone
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from loguru import logger
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine, async_sessionmaker
|
||||
|
||||
from basic_memory import db
|
||||
@@ -59,10 +58,8 @@ async def engine_factory(
|
||||
async with db.engine_session_factory(
|
||||
db_path=test_config.database_path, db_type=DatabaseType.MEMORY
|
||||
) as (engine, session_maker):
|
||||
# Initialize database
|
||||
async with db.scoped_session(session_maker) as session:
|
||||
await session.execute(text("PRAGMA foreign_keys=ON"))
|
||||
conn = await session.connection()
|
||||
# Create all tables for the DB the engine is connected to
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
|
||||
yield engine, session_maker
|
||||
@@ -311,27 +308,6 @@ async def test_graph(
|
||||
for entity in entities:
|
||||
await search_service.index_entity(entity)
|
||||
|
||||
# search_content = await entity_repository.execute_query(text("select * from search_index"),
|
||||
# use_query_options=False)
|
||||
# for row in search_content:
|
||||
# print(row)
|
||||
# print("relation:")
|
||||
# search_content = await entity_repository.execute_query(text("select * from search_index where type = 'relation'"),
|
||||
# use_query_options=False)
|
||||
# for row in search_content:
|
||||
# print(row)
|
||||
#
|
||||
# # In test_graph fixture after creating everything:
|
||||
# print("Entities:")
|
||||
# entities = await entity_repository.find_all()
|
||||
# for e in entities:
|
||||
# print(f"- {e.title} (id={e.id})")
|
||||
#
|
||||
# print("\nRelations:")
|
||||
# relations = await relation_repository.find_all()
|
||||
# for r in relations:
|
||||
# print(f"- {r.from_id} -> {r.to_id} ({r.relation_type})")
|
||||
|
||||
return {
|
||||
"root": root,
|
||||
"connected1": connected_1,
|
||||
|
||||
@@ -236,3 +236,25 @@ async def test_write_note_verbose(app):
|
||||
assert entity.relations[0].from_id == "test/test-note"
|
||||
assert entity.relations[0].to_id is None
|
||||
assert entity.relations[0].to_name == "Knowledge"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_note_memory_url(app):
|
||||
"""Test reading a note using a memory:// URL.
|
||||
|
||||
Should:
|
||||
- Handle memory:// URLs correctly
|
||||
- Normalize the URL before resolving
|
||||
- Return the note content
|
||||
"""
|
||||
# First create a note
|
||||
permalink = await notes.write_note(
|
||||
title="Memory URL Test",
|
||||
folder="test",
|
||||
content="Testing memory:// URL handling",
|
||||
)
|
||||
|
||||
# Should be able to read it with a memory:// URL
|
||||
memory_url = f"memory://{permalink}"
|
||||
content = await notes.read_note(memory_url)
|
||||
assert "Testing memory:// URL handling" in content
|
||||
|
||||
@@ -51,16 +51,17 @@ def test_relation_in_validation():
|
||||
def test_relation_response():
|
||||
"""Test RelationResponse validation."""
|
||||
data = {
|
||||
"permalink": "test/123/relates_to/test/456",
|
||||
"from_id": "test/123",
|
||||
"to_id": "test/456",
|
||||
"relation_type": "test",
|
||||
"relation_type": "relates_to",
|
||||
"from_entity": {"permalink": "test/123"},
|
||||
"to_entity": {"permalink": "test/456"},
|
||||
}
|
||||
relation = RelationResponse.model_validate(data)
|
||||
assert relation.from_id == "test/123"
|
||||
assert relation.to_id == "test/456"
|
||||
assert relation.relation_type == "test"
|
||||
assert relation.relation_type == "relates_to"
|
||||
assert relation.context is None
|
||||
|
||||
|
||||
@@ -73,16 +74,27 @@ def test_entity_out_from_attributes():
|
||||
"file_path": "test",
|
||||
"entity_type": "knowledge",
|
||||
"content_type": "text/markdown",
|
||||
"observations": [{"id": 1, "category": "note", "content": "test obs", "context": None}],
|
||||
"relations": [
|
||||
"observations": [
|
||||
{
|
||||
"id": 1,
|
||||
"from_id": "test/test",
|
||||
"to_id": "test/test",
|
||||
"relation_type": "test",
|
||||
"permalink": "permalink",
|
||||
"category": "note",
|
||||
"content": "test obs",
|
||||
"context": None,
|
||||
}
|
||||
],
|
||||
"relations": [
|
||||
{
|
||||
"id": 1,
|
||||
"permalink": "test/test/relates_to/test/test",
|
||||
"from_id": "test/test",
|
||||
"to_id": "test/test",
|
||||
"relation_type": "relates_to",
|
||||
"context": None,
|
||||
}
|
||||
],
|
||||
"created_at": "2023-01-01T00:00:00",
|
||||
"updated_at": "2023-01-01T00:00:00",
|
||||
}
|
||||
entity = EntityResponse.model_validate(db_data)
|
||||
assert entity.permalink == "test/test"
|
||||
|
||||
@@ -10,7 +10,17 @@ from basic_memory.services.link_resolver import LinkResolver
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def test_entities(entity_service, file_service):
|
||||
"""Create a set of test entities."""
|
||||
"""Create a set of test entities.
|
||||
|
||||
├── components
|
||||
│ ├── Auth Service.md
|
||||
│ └── Core Service.md
|
||||
├── config
|
||||
│ └── Service Config.md
|
||||
└── specs
|
||||
└── Core Features.md
|
||||
|
||||
"""
|
||||
|
||||
e1, _ = await entity_service.create_or_update_entity(
|
||||
EntitySchema(
|
||||
@@ -40,6 +50,20 @@ async def test_entities(entity_service, file_service):
|
||||
folder="specs",
|
||||
)
|
||||
)
|
||||
e5, _ = await entity_service.create_or_update_entity(
|
||||
EntitySchema(
|
||||
title="Sub Features 1",
|
||||
entity_type="specs",
|
||||
folder="specs/subspec",
|
||||
)
|
||||
)
|
||||
e6, _ = await entity_service.create_or_update_entity(
|
||||
EntitySchema(
|
||||
title="Sub Features 2",
|
||||
entity_type="specs",
|
||||
folder="specs/subspec",
|
||||
)
|
||||
)
|
||||
|
||||
return [e1, e2, e3, e4]
|
||||
|
||||
@@ -80,6 +104,7 @@ async def test_fuzzy_title_match_misspelling(link_resolver):
|
||||
async def test_fuzzy_title_partial_match(link_resolver):
|
||||
# Test partial match
|
||||
result = await link_resolver.resolve_link("Auth Serv")
|
||||
assert result is not None, "Did not find partial match"
|
||||
assert result.permalink == "components/auth-service"
|
||||
|
||||
|
||||
@@ -114,31 +139,3 @@ async def test_resolve_none(link_resolver):
|
||||
"""Test resolving non-existent entity."""
|
||||
# Basic new entity
|
||||
assert await link_resolver.resolve_link("New Feature") is None
|
||||
|
||||
|
||||
@pytest.mark.skip("Advanced relevance scoring not yet implemented")
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_matches_resolution(link_resolver):
|
||||
"""Test resolution when multiple potential matches exist."""
|
||||
# Add some similar entities
|
||||
test_cases = [
|
||||
{
|
||||
"link": "Service", # Ambiguous
|
||||
"expected_prefix": "components/", # Should prefer component directory match
|
||||
},
|
||||
{
|
||||
"link": "Core", # Ambiguous
|
||||
"expected_prefix": "specs/", # Should prefer specs directory match
|
||||
},
|
||||
{
|
||||
"link": "Service",
|
||||
"expected": "components/core-service", # Should pick shortest/highest scored
|
||||
},
|
||||
]
|
||||
|
||||
for case in test_cases:
|
||||
result = await link_resolver.resolve_link(case["link"])
|
||||
if "expected_prefix" in case:
|
||||
assert result.startswith(case["expected_prefix"])
|
||||
else:
|
||||
assert result == case["expected"]
|
||||
|
||||
@@ -32,21 +32,24 @@ async def test_search_permalink_observations_wildcard(search_service, test_graph
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_permalink_relation_wildcard(search_service, test_graph):
|
||||
"""Pattern matching"""
|
||||
results = await search_service.search(SearchQuery(permalink_match="test/root/connects_to/*"))
|
||||
results = await search_service.search(SearchQuery(permalink_match="test/root/connects-to/*"))
|
||||
assert len(results) == 1
|
||||
permalinks = {r.permalink for r in results}
|
||||
assert "test/root/connects-to/test/connected-entity-1" in permalinks
|
||||
|
||||
|
||||
@pytest.mark.skip("search prefix see:'https://sqlite.org/fts5.html#FTS5 Prefix Queries'")
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_permalink_wildcard2(search_service, test_graph):
|
||||
"""Pattern matching"""
|
||||
results = await search_service.search(SearchQuery(permalink_match="test/connected*"))
|
||||
assert len(results) == 2
|
||||
results = await search_service.search(
|
||||
SearchQuery(
|
||||
permalink_match="test/connected*",
|
||||
)
|
||||
)
|
||||
assert len(results) >= 2
|
||||
permalinks = {r.permalink for r in results}
|
||||
assert "test/connected1" in permalinks
|
||||
assert "test/connected2" in permalinks
|
||||
assert "test/connected-entity-1" in permalinks
|
||||
assert "test/connected-entity-2" in permalinks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -68,17 +71,27 @@ async def test_search_title(search_service, test_graph):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_search_features(search_service, test_graph):
|
||||
async def test_text_search_case_insensitive(search_service, test_graph):
|
||||
"""Test text search functionality."""
|
||||
# Case insensitive
|
||||
results = await search_service.search(SearchQuery(text="ENTITY"))
|
||||
assert any("test/root" in r.permalink for r in results)
|
||||
|
||||
# Partial word match
|
||||
results = await search_service.search(SearchQuery(text="Connect"))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_search_content_word_match(search_service, test_graph):
|
||||
"""Test text search functionality."""
|
||||
|
||||
# content word match
|
||||
results = await search_service.search(SearchQuery(text="Connected"))
|
||||
assert len(results) > 0
|
||||
assert any(r.file_path == "test/Connected Entity 2.md" for r in results)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_search_multiple_terms(search_service, test_graph):
|
||||
"""Test text search functionality."""
|
||||
|
||||
# Multiple terms
|
||||
results = await search_service.search(SearchQuery(text="root note"))
|
||||
assert any("test/root" in r.permalink for r in results)
|
||||
@@ -88,15 +101,20 @@ async def test_text_search_features(search_service, test_graph):
|
||||
async def test_pattern_matching(search_service, test_graph):
|
||||
"""Test pattern matching with various wildcards."""
|
||||
# Test wildcards
|
||||
results = await search_service.search(SearchQuery(permalink="test/*"))
|
||||
results = await search_service.search(SearchQuery(permalink_match="test/*"))
|
||||
for r in results:
|
||||
assert "test/" in r.permalink
|
||||
|
||||
# Test start wildcards
|
||||
results = await search_service.search(SearchQuery(permalink="*/observations"))
|
||||
results = await search_service.search(SearchQuery(permalink_match="*/observations"))
|
||||
for r in results:
|
||||
assert "/observations" in r.permalink
|
||||
|
||||
# Test permalink partial match
|
||||
results = await search_service.search(SearchQuery(permalink_match="test"))
|
||||
for r in results:
|
||||
assert "test/" in r.permalink
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filters(search_service, test_graph):
|
||||
|
||||
Reference in New Issue
Block a user