From 65f42d6bbbe675d57a703b8e0e162d9d53c2619a Mon Sep 17 00:00:00 2001 From: gwen Date: Sat, 29 Nov 2025 07:35:29 +0100 Subject: [PATCH] first commit --- .gitignore | 144 +++++++++++ DEVELOPMENT.md | 185 +++++++++++++ README.md | 235 +++++++++++++++++ TESTING.md | 110 ++++++++ dfdr/__init__.py | 20 ++ dfdr/checksum.py | 125 +++++++++ dfdr/cli.py | 306 ++++++++++++++++++++++ dfdr/config.py | 144 +++++++++++ dfdr/exceptions.py | 37 +++ dfdr/fetcher.py | 115 +++++++++ dfdr/storage.py | 474 ++++++++++++++++++++++++++++++++++ essai/analytics/metrics.csv | 3 + essai/analytics/settings.yaml | 53 ++++ essai/analytics/users.json | 60 +++++ essai/sample.json | 16 ++ examples/basic_usage.py | 52 ++++ install.sh | 1 + licence.txt | 19 ++ readme.rst | 165 ++++++++++++ requirements.txt | 10 + setup.py | 47 ++++ tests/__init__.py | 3 + tests/test_basic.py | 204 +++++++++++++++ todo.txt | 12 + 24 files changed, 2540 insertions(+) create mode 100644 .gitignore create mode 100644 DEVELOPMENT.md create mode 100644 README.md create mode 100644 TESTING.md create mode 100644 dfdr/__init__.py create mode 100644 dfdr/checksum.py create mode 100644 dfdr/cli.py create mode 100644 dfdr/config.py create mode 100644 dfdr/exceptions.py create mode 100644 dfdr/fetcher.py create mode 100644 dfdr/storage.py create mode 100644 essai/analytics/metrics.csv create mode 100644 essai/analytics/settings.yaml create mode 100644 essai/analytics/users.json create mode 100644 essai/sample.json create mode 100644 examples/basic_usage.py create mode 100755 install.sh create mode 100644 licence.txt create mode 100644 readme.rst create mode 100644 requirements.txt create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/test_basic.py create mode 100644 todo.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49d4666 --- /dev/null +++ b/.gitignore @@ -0,0 +1,144 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# dfdr specific +.dfdr/ +*.dfdr + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..293deff --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,185 @@ +# dfdr Development Guide + +## Project Structure + +``` +dfdr/ +├── .gitignore # Git ignore patterns +├── Makefile # Development automation +├── README.rst # Main documentation +├── requirements.txt # Python dependencies +├── setup.py # Package configuration +├── DEVELOPMENT.md # This file +├── dfdr/ # Main package +│ ├── __init__.py # Package initialization +│ ├── cli.py # Command-line interface +│ ├── config.py # Configuration management +│ ├── checksum.py # MD5 checksum utilities +│ ├── exceptions.py # Custom exceptions +│ ├── fetcher.py # HTTP client with httpx +│ └── storage.py # Storage and working copy management +├── tests/ # Test suite +│ ├── __init__.py +│ └── test_basic.py # Basic functionality tests +└── examples/ # Usage examples + └── basic_usage.py # Programmatic usage example +``` + +## Installation + +1. Create and activate a virtual environment: +```bash +python -m venv .venv +source .venv/bin/activate # On Windows: .venv\Scripts\activate +``` + +2. Install in development mode: +```bash +make install-dev +# or manually: +pip install -e . +pip install pytest pytest-cov black flake8 mypy +``` + +## Development Workflow + +### Running Tests +```bash +make test # Run basic tests +make test-cov # Run tests with coverage report +``` + +### Code Quality +```bash +make format # Format code with black +make lint # Run linting with flake8 and mypy +``` + +### Building and Distribution +```bash +make build # Build distribution packages +make clean # Clean build artifacts +``` + +## Command Reference + +### Phase 1 Commands (Implemented) +- `dfdr init` - Initialize a dfdr repository +- `dfdr remote add ` - Add remote data registry +- `dfdr remote list` - List configured remotes +- `dfdr remote remove ` - Remove a remote +- `dfdr add :` - Add file from remote to working copy +- `dfdr fetch` - Fetch all files from remotes to storage +- `dfdr pull [file]` - Update working copy from storage +- `dfdr status` - Show file sync status + +### Usage Examples + +```bash +# Initialize repository +dfdr init + +# Add a remote data registry +dfdr remote add mydata https://data.example.com/ + +# Add specific files +dfdr add mydata:datasets/sales.csv +dfdr add mydata:models/config.json + +# Fetch all available files to local storage +dfdr fetch + +# Check status +dfdr status + +# Update working copy +dfdr pull +``` + +## Architecture + +### Core Components + +1. **Config** (`config.py`): Manages remote registries and local configuration +2. **Fetcher** (`fetcher.py`): HTTP client for downloading files and indices +3. **Storage** (`storage.py`): Manages local storage and working copy operations +4. **Checksum** (`checksum.py`): MD5 checksum calculation and verification +5. **CLI** (`cli.py`): Command-line interface using Click and Rich + +### Data Flow + +1. **Remote Registry**: HTTP server with `index.json` files listing available files +2. **Local Storage**: `.dfdr/storage/` mirrors remote files +3. **Working Copy**: Project files with corresponding `.dfdr` checksum files +4. **Configuration**: `.dfdr/config.json` stores remote registry URLs + +### File Structure + +``` +project/ +├── .dfdr/ +│ ├── config.json # Remote configurations +│ └── storage/ # Local mirror of remote files +│ └── remote_name/ +│ └── file.csv +├── data_file.csv # Working copy file +└── data_file.csv.dfdr # MD5 checksum +``` + +## Protocol Specification + +### Remote Data Registry + +Remote registries must serve files over HTTP with the following structure: + +1. **Index Files**: Each directory contains `index.json`: +```json +{ + "files": ["file1.csv", "file2.json", "subfolder/file3.yaml"] +} +``` + +2. **File Access**: Files are accessible via direct HTTP GET requests +3. **Content Types**: Supports CSV, JSON, YAML, TXT files + +### Local Storage + +1. **Checksums**: Each data file has a corresponding `.dfdr` file containing MD5 hash +2. **Storage Mirror**: `.dfdr/storage/` contains exact copies of remote files +3. **Configuration**: JSON configuration file tracks remote registries + +## Testing + +Run the test suite: +```bash +python -m pytest tests/ -v +``` + +Test coverage: +```bash +python -m pytest tests/ --cov=dfdr --cov-report=html +``` + +## Contributing + +1. Fork the repository +2. Create a feature branch +3. Make changes with tests +4. Run `make format` and `make lint` +5. Submit a pull request + +## Future Enhancements + +### Phase 2 Features +- Directory synchronization +- Nested folder support with recursive index.json +- File metadata tracking (size, modification time) +- Conflict resolution strategies +- Incremental updates based on ETags/Last-Modified headers + +### Advanced Features +- Authentication support for private registries +- Compression support +- Parallel downloads +- Registry mirroring +- Plugin system for custom protocols diff --git a/README.md b/README.md new file mode 100644 index 0000000..cd6a74d --- /dev/null +++ b/README.md @@ -0,0 +1,235 @@ +# DefDer (Data Registry Definition) - dfdr + +A Python command-line tool for managing local data registries, inspired by DVC and Git but focused specifically on data registry functionality. + +## Overview + +dfdr allows you to: + +- Declare local data sources (folders containing CSV, JSON, YAML, TXT files) +- Add specific files from data registries to your working copy +- Mirror data locally for efficient access +- Track file changes with checksums +- Keep your working copy synchronized with local registries + +## Installation + +### Prerequisites + +- Python 3.8 or higher +- pip (Python package installer) + +### Install from PyPI + +```bash +pip install dfdr +``` + +### Install from source + +```bash +git clone git@defder.fr:dfdr.git +cd dfdr +pip install -e . +``` + +## Quick Start + +1. Initialize a dfdr repository: + +```bash +dfdr init +``` + +2. Add a local data registry: + +```bash +dfdr registry add myregistry /path/to/local/data/folder +``` + +3. Add files from the registry to your working copy: + +```bash +dfdr add myregistry:datasets/sales.csv +dfdr add myregistry:models/config.json +``` + +4. Add a file to a specific subdirectory: + +```bash +dfdr add myregistry:datasets/customers.csv -d ./data/customers +``` + +5. Update your local cache: + +```bash +dfdr fetch +``` + +6. Check the status of your files: + +```bash +dfdr status +``` + +7. Update your working copy: + +```bash +dfdr pull +``` + +8. Push changes back to the registry: + +```bash +dfdr push datasets/sales.csv +``` + +9. Move a file within the working copy: + +```bash +dfdr move models/config.json ./configs/model_config.json +``` + +## Detailed Usage + +### Managing Local Registries + +```bash +# Add a local registry +dfdr registry add production /path/to/production/data + +# List all registries +dfdr registry list + +# Remove a registry +dfdr registry remove staging +``` + +### Working with Files + +```bash +# Add a specific file +dfdr add production:datasets/customer_data.csv + +# Add a file to a specific subdirectory +dfdr add production:datasets/sales.csv -d ./data/sales + +# Add an entire directory +dfdr add production:models + +# Fetch all data to local cache +dfdr fetch + +# Update working copy (all files) +dfdr pull + +# Update a specific file +dfdr pull datasets/customer_data.csv + +# Check status of all tracked files +dfdr status + +# Push changes back to the registry +dfdr push datasets/customer_data.csv + +# Move a file within the working copy +dfdr move models/config.json ./configs/model_config.json +``` + +## Command Reference + +### `dfdr init` + +Initialize a new dfdr repository in the current directory. + +### `dfdr registry` + +Manage local data registries. + +- `add `: Add a local data registry +- `list`: List all configured registries +- `remove `: Remove a local data registry + +### `dfdr add : [-d ]` + +Add a specific file or directory from a registry to your working copy. Optionally specify a destination subdirectory. + +### `dfdr fetch` + +Mirror all data to local `.dfdr` storage from all remotes. + +### `dfdr pull [file_path]` + +Update working copy from cache (all files or specific file). + +### `dfdr status` + +Show sync status of files, including origin information. + +### `dfdr push ` + +Push changes in a file back to its origin data registry. + +### `dfdr move ` + +Move a tracked file to a new destination within the working copy. + +## Data registry structure + +Local data registries should follow this structure: + +- files are stored in local directories +- the tool will automatically discover files in the directory + +For example: + +``` +/path/to/registry/ +├── datasets/ +│ ├── sales.csv +│ └── customers.json +└── models/ + └── config.yaml +``` + +## Architecture + +dfdr is designed with a modular architecture: + +- `cli.py`: Command-line interface using Click and Rich for display +- `config.py`: Configuration management (local registries, paths) +- `storage.py`: Local storage management and synchronization +- `fetcher.py`: Local file system operations +- `checksum.py`: MD5 checksum calculation and verification +- `exceptions.py`: Custom exception handling + +## Local Storage + +dfdr creates a `.dfdr` directory in your project containing: + +- `config.json`: Local registry configuration +- `storage/`: Local mirror of registry data +- `*.dfdr` files: YAML files containing MD5 checksums and origin information for each data file (e.g., `sales.csv.dfdr`) + +The `.dfdr` files now include additional information: +- MD5 checksum of the file +- Registry name (origin) +- Original path in the registry + +This enhanced structure allows for better tracking and management of files across different registries. + +### Development Setup + +1. Clone the repository +2. Create a virtual environment: `python -m venv venv` +3. Activate the virtual environment: + - On Windows: `venv\Scripts\activate` + - On macOS and Linux: `source venv/bin/activate` +4. Install development dependencies: `pip install -r requirements.txt` +5. Install the package in editable mode: `pip install -e .` + +## License + +This project is licensed under the MIT License - see the [LICENSE](licence.txt) file for details. + +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) diff --git a/TESTING.md b/TESTING.md new file mode 100644 index 0000000..7320395 --- /dev/null +++ b/TESTING.md @@ -0,0 +1,110 @@ +# Testing dfdr + +This document outlines the test cases for verifying the functionality of the dfdr tool. + +## Test Cases + +### 1. Relative Paths for Registry List + +- Add a registry using a relative path: + ``` + dfdr registry add test_registry ../test_data + ``` +- List registries and verify the path is displayed as relative: + ``` + dfdr registry list + ``` +- Use the registry to add a file: + ``` + dfdr add test_registry:sample.txt + ``` + +### 2. Specifying Destination Subdirectories + +- Add a file to a specific subdirectory: + ``` + dfdr add test_registry:sample.txt -d ./subfolder + ``` +- Verify the file is placed in the correct location: + ``` + ls ./subfolder + ``` +- Check that the .dfdr file contains the correct origin information: + ``` + cat ./subfolder/sample.txt.dfdr + ``` + +### 3. Enhanced .dfdr Files with Origin Information + +- Add a file from a registry: + ``` + dfdr add test_registry:config.json + ``` +- Examine the .dfdr file: + ``` + cat config.json.dfdr + ``` +- Use the `status` command to verify the origin information: + ``` + dfdr status + ``` + +### 4. Push Functionality + +- Add a file from a registry: + ``` + dfdr add test_registry:data.csv + ``` +- Modify the file in the working copy: + ``` + echo "new data" >> data.csv + ``` +- Push changes back to the registry: + ``` + dfdr push data.csv + ``` +- Verify changes in the original registry: + ``` + cat ../test_data/data.csv + ``` + +### 5. Move Command + +- Add a file from a registry: + ``` + dfdr add test_registry:script.py + ``` +- Move the file within the working copy: + ``` + dfdr move script.py ./src/script.py + ``` +- Verify the file is moved and .dfdr file is updated: + ``` + ls ./src + cat ./src/script.py.dfdr + ``` +- Check status for correct location and origin information: + ``` + dfdr status + ``` + +### 6. Error Handling + +- Attempt to push a non-existent file: + ``` + dfdr push non_existent_file.txt + ``` +- Try to move a file to an invalid destination: + ``` + dfdr move script.py /root/invalid_destination.py + ``` +- Add a file from a non-existent registry: + ``` + dfdr add fake_registry:file.txt + ``` + +## Running the Tests + +To run these tests, set up a test environment with a sample data registry, then execute each command and verify the results match the expected behavior. + +Remember to clean up the test environment after running the tests to ensure a clean state for future test runs. diff --git a/dfdr/__init__.py b/dfdr/__init__.py new file mode 100644 index 0000000..4ec17f3 --- /dev/null +++ b/dfdr/__init__.py @@ -0,0 +1,20 @@ +""" +DefDer (Data Registry Definition) - dfdr + +A Python tool for managing remote data registries. +""" + +__version__ = "0.1.0" +__author__ = "DefDer Team" +__email__ = "contact@defder.org" + +from .exceptions import DfdrError, RemoteError, ChecksumError, ConfigError, FileSystemError, FileNotFoundError + +__all__ = [ + "DfdrError", + "RemoteError", + "ChecksumError", + "ConfigError", + "FileSystemError", + "FileNotFoundError", +] diff --git a/dfdr/checksum.py b/dfdr/checksum.py new file mode 100644 index 0000000..1586cc9 --- /dev/null +++ b/dfdr/checksum.py @@ -0,0 +1,125 @@ +""" +Checksum utilities for dfdr. +""" + +import hashlib +from pathlib import Path +from typing import Optional + +from .exceptions import ChecksumError + + +def calculate_md5(file_path: Path) -> str: + """ + Calculate MD5 checksum of a file. + + Args: + file_path: Path to the file + + Returns: + MD5 checksum as hexadecimal string + + Raises: + ChecksumError: If file cannot be read + """ + try: + hash_md5 = hashlib.md5() + with open(file_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + except (OSError, IOError) as e: + raise ChecksumError(f"Failed to calculate checksum for {file_path}: {e}") + + +import yaml + +def save_checksum(file_path: Path, checksum: str, registry_name: str, original_path: str) -> None: + """ + Save checksum and origin information to a .dfdr file. + + Args: + file_path: Path to the data file + checksum: MD5 checksum to save + registry_name: Name of the registry the file came from + original_path: Original path of the file in the registry + + Raises: + ChecksumError: If checksum file cannot be written + """ + checksum_path = Path(f"{file_path}.dfdr") + data = { + "checksum": checksum, + "registry_name": registry_name, + "original_path": original_path + } + try: + with open(checksum_path, "w", encoding="utf-8") as f: + yaml.dump(data, f) + except (OSError, IOError, yaml.YAMLError) as e: + raise ChecksumError(f"Failed to save checksum and origin info to {checksum_path}: {e}") + +def load_checksum_info(file_path: Path) -> Optional[dict]: + """ + Load checksum and origin information from a .dfdr file. + + Args: + file_path: Path to the data file + + Returns: + Dictionary containing checksum, registry_name, and original_path if file exists, None otherwise + + Raises: + ChecksumError: If checksum file exists but cannot be read or parsed + """ + checksum_path = Path(f"{file_path}.dfdr") + if not checksum_path.exists(): + return None + + try: + with open(checksum_path, "r", encoding="utf-8") as f: + data = yaml.safe_load(f) + return data + except (OSError, IOError, yaml.YAMLError) as e: + raise ChecksumError(f"Failed to read checksum and origin info from {checksum_path}: {e}") + + +def load_checksum(file_path: Path) -> Optional[str]: + """ + Load checksum from a .dfdr file. + + Args: + file_path: Path to the data file + + Returns: + MD5 checksum if file exists, None otherwise + + Raises: + ChecksumError: If checksum file exists but cannot be read + """ + info = load_checksum_info(file_path) + return info["checksum"] if info else None + + +def verify_checksum(file_path: Path) -> bool: + """ + Verify that a file's checksum matches its .dfdr file. + + Args: + file_path: Path to the data file + + Returns: + True if checksums match, False if they don't or .dfdr file doesn't exist + + Raises: + ChecksumError: If files cannot be read + """ + if not file_path.exists(): + return False + + info = load_checksum_info(file_path) + if info is None or "checksum" not in info: + return False + + current_checksum = calculate_md5(file_path) + return info["checksum"] == current_checksum diff --git a/dfdr/cli.py b/dfdr/cli.py new file mode 100644 index 0000000..7be62e4 --- /dev/null +++ b/dfdr/cli.py @@ -0,0 +1,306 @@ +""" +Command-line interface for dfdr. +""" + +import sys +from pathlib import Path + +import click +from rich.console import Console +from rich.table import Table +from rich.text import Text + +from .config import Config +from .storage import Storage +from .exceptions import DfdrError + + +console = Console() + + +def handle_error(func): + """Decorator to handle exceptions and display user-friendly error messages.""" + import functools + + @functools.wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except DfdrError as e: + console.print(f"[red]Error:[/red] {e}") + sys.exit(1) + except Exception as e: + console.print(f"[red]Unexpected error:[/red] {e}") + sys.exit(1) + return wrapper + + +@click.group() +@click.version_option(version="0.1.0", prog_name="dfdr") +def cli(): + """ + DefDer (Data Registry Definition) - dfdr + + A tool for managing local data registries. + """ + pass + + +@cli.group() +def registry(): + """Manage local data registries.""" + pass + + +@registry.command("add") +@click.argument("name") +@click.argument("path") +@handle_error +def registry_add(name, path): + """Add a local data registry.""" + config = Config() + config.add_remote(name, path) + console.print(f"[green]✓[/green] Added local registry '{name}' -> {path}") + + +@registry.command("list") +@handle_error +def registry_list(): + """List all configured local registries.""" + config = Config() + registries = config.list_remotes() + + if not registries: + console.print("No local registries configured.") + return + + table = Table(title="Configured Local Registries") + table.add_column("Name", style="cyan") + table.add_column("Path", style="blue") + + for name, path in registries.items(): + table.add_row(name, str(path)) + + console.print(table) + + +@registry.command("remove") +@click.argument("name") +@handle_error +def registry_remove(name): + """Remove a local data registry.""" + config = Config() + config.remove_remote(name) + console.print(f"[green]✓[/green] Removed local registry '{name}'") + + +@cli.command() +@click.argument("file_spec") +@click.option("--destination", "-d", help="Destination path in working copy") +@handle_error +def add(file_spec, destination): + """ + Add a file or directory from local registry to working copy. + + FILE_SPEC should be in format: registry_name:file_path + + Examples: + dfdr add myregistry:sample.json # Add a single file + dfdr add myregistry:analytics # Add entire directory + dfdr add myregistry:sample.json -d ./subfolder # Add to specific destination + """ + if ":" not in file_spec: + raise click.BadParameter("File spec must be in format 'registry_name:file_path'") + + registry_name, file_path = file_spec.split(":", 1) + + config = Config() + storage = Storage(config) + + with console.status(f"[bold green]Adding {file_path} from {registry_name}..."): + storage.add_file(registry_name, file_path, destination) + + console.print(f"[green]✓[/green] Added {file_path} from {registry_name}") + + +@cli.command() +@handle_error +def fetch(): + """Fetch all files from local registries to local storage.""" + config = Config() + storage = Storage(config) + + with console.status("[bold green]Fetching files from local registries..."): + fetched_files = storage.fetch_all() + + total_files = sum(len(files) for files in fetched_files.values()) + + if total_files == 0: + console.print("No files fetched.") + return + + console.print(f"[green]✓[/green] Fetched {total_files} files") + + for registry_name, files in fetched_files.items(): + if files: + console.print(f" {registry_name}: {len(files)} files") + + +@cli.command() +@click.argument("file_path", required=False) +@handle_error +def pull(file_path): + """ + Update working copy from storage. + + If FILE_PATH is specified, only that file is updated. + Otherwise, all files are updated. + """ + config = Config() + storage = Storage(config) + + if file_path: + with console.status(f"[bold green]Pulling {file_path}..."): + updated = storage.pull_file(file_path) + + if updated: + console.print(f"[green]✓[/green] Updated {file_path}") + else: + console.print(f"[yellow]•[/yellow] {file_path} is already up-to-date") + else: + with console.status("[bold green]Pulling all files..."): + updated_files = storage.pull_all() + + if updated_files: + console.print(f"[green]✓[/green] Updated {len(updated_files)} files") + for file in updated_files: + console.print(f" {file}") + else: + console.print("[yellow]•[/yellow] All files are up-to-date") + + +@cli.command() +@handle_error +def status(): + """Show status of tracked files.""" + config = Config() + storage = Storage(config) + + file_status = storage.get_status() + + if not file_status: + console.print("No tracked files.") + return + + # Group files by status + status_groups = { + "up-to-date": [], + "modified": [], + "missing": [], + "untracked": [] + } + + for file_path, status in file_status.items(): + status_groups[status].append(file_path) + + # Display results + if status_groups["up-to-date"]: + console.print(f"[green]Up-to-date files ({len(status_groups['up-to-date'])}):[/green]") + for file in status_groups["up-to-date"]: + info = storage.get_file_info(file) + console.print(f" [green]✓[/green] {file} (from {info['registry_name']}:{info['original_path']})") + console.print() + + if status_groups["modified"]: + console.print(f"[yellow]Modified files ({len(status_groups['modified'])}):[/yellow]") + for file in status_groups["modified"]: + info = storage.get_file_info(file) + console.print(f" [yellow]M[/yellow] {file} (from {info['registry_name']}:{info['original_path']})") + console.print() + + if status_groups["missing"]: + console.print(f"[red]Missing files ({len(status_groups['missing'])}):[/red]") + for file in status_groups["missing"]: + info = storage.get_file_info(file) + console.print(f" [red]![/red] {file} (from {info['registry_name']}:{info['original_path']})") + console.print() + + if status_groups["untracked"]: + console.print(f"[blue]Untracked files ({len(status_groups['untracked'])}):[/blue]") + for file in status_groups["untracked"]: + console.print(f" [blue]?[/blue] {file}") + + +@cli.command() +@handle_error +def init(): + """Initialize a new dfdr repository in the current directory.""" + config = Config() + + if config.is_initialized(): + console.print("[yellow]Repository already initialized.[/yellow]") + return + + # Config initialization happens automatically in Config.__init__() + console.print(f"[green]✓[/green] Initialized dfdr repository in {config.project_root}") + +@cli.command() +@handle_error +def validate(): + """Validate all configured local registries.""" + config = Config() + + with console.status("[bold green]Validating local registries..."): + config.validate_local_registries() + + console.print("[green]✓[/green] All local registries are valid.") + + +@cli.command() +@click.argument("file_path") +@handle_error +def push(file_path): + """ + Push changes in a file back to its origin data registry. + + FILE_PATH: Path of the file to push (relative to project root) + """ + config = Config() + storage = Storage(config) + + with console.status(f"[bold green]Pushing {file_path}..."): + pushed = storage.push_file(file_path) + + if pushed: + console.print(f"[green]✓[/green] Pushed changes in {file_path} to origin") + else: + console.print(f"[yellow]•[/yellow] No changes to push for {file_path}") + + +@cli.command() +@click.argument("file_path") +@click.argument("new_destination") +@handle_error +def move(file_path, new_destination): + """ + Move a tracked file to a new destination within the working copy. + + FILE_PATH: Current path of the file (relative to project root) + NEW_DESTINATION: New destination path for the file (relative to project root) + """ + config = Config() + storage = Storage(config) + + with console.status(f"[bold green]Moving {file_path} to {new_destination}..."): + storage.update_file_destination(file_path, new_destination) + + console.print(f"[green]✓[/green] Moved {file_path} to {new_destination}") + + +def main(): + """Main entry point for the CLI.""" + cli() + + +if __name__ == "__main__": + main() diff --git a/dfdr/config.py b/dfdr/config.py new file mode 100644 index 0000000..b0584f8 --- /dev/null +++ b/dfdr/config.py @@ -0,0 +1,144 @@ +""" +Configuration management for dfdr. +""" + +import json +from pathlib import Path +from typing import Dict, Optional + +from .exceptions import ConfigError + + +class Config: + """Manages dfdr configuration.""" + + def __init__(self, project_root: Optional[Path] = None): + """ + Initialize configuration. + + Args: + project_root: Root directory of the project. If None, uses current directory. + """ + self.project_root = project_root or Path.cwd() + self.dfdr_dir = self.project_root / ".dfdr" + self.config_file = self.dfdr_dir / "config.json" + self.storage_dir = self.dfdr_dir / "storage" + + # Ensure directories exist + self.dfdr_dir.mkdir(exist_ok=True) + self.storage_dir.mkdir(exist_ok=True) + + self._config = self._load_config() + + def _load_config(self) -> Dict: + """Load configuration from file.""" + if not self.config_file.exists(): + return {"remotes": {}} + + try: + with open(self.config_file, "r", encoding="utf-8") as f: + return json.load(f) + except (json.JSONDecodeError, OSError, IOError) as e: + raise ConfigError(f"Failed to load config from {self.config_file}: {e}") + + def _save_config(self) -> None: + """Save configuration to file.""" + try: + with open(self.config_file, "w", encoding="utf-8") as f: + json.dump(self._config, f, indent=2) + except (OSError, IOError) as e: + raise ConfigError(f"Failed to save config to {self.config_file}: {e}") + + def add_remote(self, name: str, path: str) -> None: + """ + Add a local registry. + + Args: + name: Name of the local registry + path: Path to the local registry + + Raises: + ConfigError: If local registry already exists or path is invalid + """ + if name in self._config["remotes"]: + raise ConfigError(f"Local registry '{name}' already exists") + + local_path = Path(path).resolve() + if not local_path.is_dir(): + raise ConfigError(f"Invalid path for local registry: {path}") + + self._config["remotes"][name] = {"path": str(local_path)} + self._save_config() + + def remove_remote(self, name: str) -> None: + """ + Remove a local registry. + + Args: + name: Name of the local registry to remove + + Raises: + ConfigError: If local registry doesn't exist + """ + if name not in self._config["remotes"]: + raise ConfigError(f"Local registry '{name}' does not exist") + + del self._config["remotes"][name] + self._save_config() + + def get_remote_path(self, name: str) -> Path: + """ + Get absolute path for a local registry. + + Args: + name: Name of the local registry + + Returns: + Absolute Path of the local registry + + Raises: + ConfigError: If local registry doesn't exist + """ + if name not in self._config["remotes"]: + raise ConfigError(f"Local registry '{name}' does not exist") + + return Path(self._config["remotes"][name]["path"]) + + def list_remotes(self) -> Dict[str, Path]: + """ + List all configured local registries. + + Returns: + Dictionary mapping local registry names to absolute paths + """ + return {name: Path(remote["path"]) for name, remote in self._config["remotes"].items()} + + def get_storage_path(self, remote_name: str, file_path: str) -> Path: + """ + Get local storage path for a file from a local registry. + + Args: + remote_name: Name of the local registry + file_path: Path of the file in the local registry (already relative to remote) + + Returns: + Local path where the file should be stored + """ + # file_path is already relative to the remote, so just use it directly + return self.storage_dir / remote_name / file_path + + def is_initialized(self) -> bool: + """Check if the current directory is a dfdr repository.""" + return self.dfdr_dir.exists() and self.config_file.exists() + + def validate_local_registries(self) -> None: + """ + Validate all configured local registries. + + Raises: + ConfigError: If any local registry path is invalid + """ + for name, remote in self._config["remotes"].items(): + path = Path(remote["path"]) + if not path.is_dir(): + raise ConfigError(f"Invalid path for local registry '{name}': {path}") diff --git a/dfdr/exceptions.py b/dfdr/exceptions.py new file mode 100644 index 0000000..851e3a6 --- /dev/null +++ b/dfdr/exceptions.py @@ -0,0 +1,37 @@ +""" +Custom exceptions for dfdr. +""" + + +class DfdrError(Exception): + """Base exception for all dfdr errors.""" + pass + + +class RemoteError(DfdrError): + """Exception raised for remote registry errors.""" + pass + + +class ChecksumError(DfdrError): + """Exception raised for checksum validation errors.""" + pass + + +class ConfigError(DfdrError): + """Exception raised for configuration errors.""" + pass + + +class FileNotFoundError(DfdrError): + """Exception raised when a file is not found in the registry.""" + pass + + +class NetworkError(DfdrError): + """Exception raised for network-related errors.""" + pass + +class FileSystemError(DfdrError): + """Exception raised for file system related errors.""" + pass diff --git a/dfdr/fetcher.py b/dfdr/fetcher.py new file mode 100644 index 0000000..16c5e8e --- /dev/null +++ b/dfdr/fetcher.py @@ -0,0 +1,115 @@ +""" +Local file system utilities for dfdr. +""" + +import json +from pathlib import Path +from typing import Dict, List, Optional +import shutil + +from .exceptions import FileSystemError, FileNotFoundError + + +class Fetcher: + """Handles local file system operations for data registries.""" + + def __init__(self): + """Initialize the fetcher.""" + pass + + def __enter__(self): + """Context manager entry.""" + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """Context manager exit.""" + pass + + def fetch_index(self, base_path: Path, directory: str = "") -> List[str]: + """ + List files in a local directory. + + Args: + base_path: Base path of the local registry + directory: Directory path (empty for root) + + Returns: + List of files in the directory + + Raises: + FileNotFoundError: If directory doesn't exist + FileSystemError: If there's an error reading the directory + """ + dir_path = base_path / directory + + if not dir_path.exists(): + raise FileNotFoundError(f"Directory not found: {dir_path}") + + if not dir_path.is_dir(): + raise FileSystemError(f"Path is not a directory: {dir_path}") + + try: + return [f.name for f in dir_path.iterdir() if f.is_file()] + except Exception as e: + raise FileSystemError(f"Error reading directory {dir_path}: {e}") + + def fetch_file(self, base_path: Path, file_path: str, output_path: Path) -> None: + """ + Copy a file from the local registry to the output path. + + Args: + base_path: Base path of the local registry + file_path: Path of the file to fetch + output_path: Local path to save the file + + Raises: + FileNotFoundError: If file doesn't exist + FileSystemError: If there's an error copying the file + """ + source_path = base_path / file_path + + if not source_path.exists(): + raise FileNotFoundError(f"File not found: {source_path}") + + # Ensure output directory exists + output_path.parent.mkdir(parents=True, exist_ok=True) + + try: + shutil.copy2(source_path, output_path) + except Exception as e: + raise FileSystemError(f"Failed to copy file from {source_path} to {output_path}: {e}") + + def check_file_exists(self, base_path: Path, file_path: str) -> bool: + """ + Check if a file exists in the local registry. + + Args: + base_path: Base path of the local registry + file_path: Path of the file to check + + Returns: + True if file exists, False otherwise + """ + return (base_path / file_path).is_file() + + def get_file_info(self, base_path: Path, file_path: str) -> Optional[Dict]: + """ + Get file information from local registry. + + Args: + base_path: Base path of the local registry + file_path: Path of the file + + Returns: + Dictionary with file info or None if file doesn't exist + """ + file_path = base_path / file_path + if not file_path.is_file(): + return None + + stat = file_path.stat() + return { + "size": stat.st_size, + "last_modified": stat.st_mtime, + "created": stat.st_ctime, + } diff --git a/dfdr/storage.py b/dfdr/storage.py new file mode 100644 index 0000000..5236b60 --- /dev/null +++ b/dfdr/storage.py @@ -0,0 +1,474 @@ +""" +Storage management for dfdr. +""" + +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +from .checksum import calculate_md5, save_checksum, load_checksum, verify_checksum, load_checksum_info +from .config import Config +from .fetcher import Fetcher +from .exceptions import DfdrError, FileNotFoundError, FileSystemError + + +class Storage: + """Manages local storage and working copy operations.""" + + def __init__(self, config: Config): + """ + Initialize storage manager. + + Args: + config: Configuration instance + """ + self.config = config + + def add_file(self, remote_name: str, file_path: str, destination: Optional[str] = None) -> None: + """ + Add a file or directory from local registry to working copy. + + Args: + remote_name: Name of the local registry + file_path: Path of the file or directory in the local registry + destination: Optional destination path in the working copy + + Raises: + DfdrError: If operation fails + """ + # Get remote path + remote_path = Path(self.config.get_remote_path(remote_name)) + full_remote_path = (remote_path / file_path).resolve() + + if not full_remote_path.exists(): + raise FileNotFoundError(f"File or directory '{file_path}' does not exist in the remote path '{remote_path}'") + + if full_remote_path.is_file(): + self._add_single_file(remote_name, remote_path, file_path, None, destination) + elif full_remote_path.is_dir(): + directory_files = [f.name for f in full_remote_path.iterdir() if f.is_file()] + self._add_directory(remote_name, remote_path, file_path, directory_files, None, destination) + else: + raise ValueError(f"'{file_path}' is neither a file nor a directory") + + def _add_single_file(self, remote_name: str, remote_path: Path, file_path: str, fetcher: Optional[Fetcher], destination: Optional[str] = None) -> None: + """Add a single file from local registry to working copy.""" + # Resolve both paths to ensure consistent comparison + resolved_remote_path = remote_path.resolve() + full_remote_path = (remote_path / file_path).resolve() + + if not full_remote_path.exists(): + raise FileNotFoundError(f"File '{file_path}' does not exist in the remote path '{remote_path}'") + + if not full_remote_path.is_file(): + raise ValueError(f"'{file_path}' is not a file") + + try: + relative_path = full_remote_path.relative_to(resolved_remote_path) + except ValueError: + raise ValueError(f"'{full_remote_path}' is not in the subpath of '{resolved_remote_path}'") + + filename = relative_path.name + + if destination: + working_copy_path = self.config.project_root / destination / filename + else: + working_copy_path = self.config.project_root / filename + + storage_path = self.config.get_storage_path(remote_name, str(relative_path)) + + # Copy file to storage + storage_path.parent.mkdir(parents=True, exist_ok=True) + with open(full_remote_path, "rb") as src, open(storage_path, "wb") as dst: + dst.write(src.read()) + + # Copy to working copy + working_copy_path.parent.mkdir(parents=True, exist_ok=True) + with open(storage_path, "rb") as src, open(working_copy_path, "wb") as dst: + dst.write(src.read()) + + # Calculate and save checksum with origin information + checksum = calculate_md5(working_copy_path) + save_checksum(working_copy_path, checksum, remote_name, str(relative_path)) + + + def _add_directory(self, remote_name: str, remote_path: Path, dir_path: str, files: List[str], fetcher: Optional[Fetcher], destination: Optional[str] = None) -> None: + """Add all files from a directory to working copy.""" + # Create the directory in working copy + if destination: + working_copy_dir = self.config.project_root / destination / Path(dir_path).name + else: + working_copy_dir = self.config.project_root / Path(dir_path).name + working_copy_dir.mkdir(parents=True, exist_ok=True) + + for file_name in files: + # Full path in local registry + full_file_path = f"{dir_path}/{file_name}" + full_remote_path = remote_path / full_file_path + # Storage path (preserves full structure) + storage_path = self.config.get_storage_path(remote_name, full_file_path) + + # Working copy path (preserves directory structure) + if destination: + working_copy_path = working_copy_dir / file_name + else: + working_copy_path = working_copy_dir / file_name + + try: + # Copy file to storage + storage_path.parent.mkdir(parents=True, exist_ok=True) + with open(full_remote_path, "rb") as src, open(storage_path, "wb") as dst: + dst.write(src.read()) + + # Copy to working copy + working_copy_path.parent.mkdir(parents=True, exist_ok=True) + with open(storage_path, "rb") as src, open(working_copy_path, "wb") as dst: + dst.write(src.read()) + + # Calculate and save checksum with origin information + checksum = calculate_md5(working_copy_path) + save_checksum(working_copy_path, checksum, remote_name, full_file_path) + + + except Exception as e: + pass # Silently skip failed files + + def fetch_all(self) -> Dict[str, List[str]]: + """ + Fetch all files from all local registries to storage. + Also fetches files from subdirectories that have been previously added. + + Returns: + Dictionary mapping local registry names to lists of fetched files + + Raises: + DfdrError: If operation fails + """ + fetched_files = {} + + with Fetcher() as fetcher: + for remote_name, remote_path_str in self.config.list_remotes().items(): + remote_path = Path(remote_path_str) + fetched_files[remote_name] = [] + + try: + # Get root index + files = fetcher.fetch_index(remote_path) + + for file_path in files: + storage_path = self.config.get_storage_path(remote_name, file_path) + + try: + fetcher.fetch_file(remote_path, file_path, storage_path) + fetched_files[remote_name].append(file_path) + except Exception as e: + # Continue with other files if one fails + pass # Silently skip failed files + + # Also fetch from subdirectories that have been previously added + # Look for mapping files to find directories we've added + remote_storage = self.config.storage_dir / remote_name + if remote_storage.exists(): + for mapping_file in remote_storage.rglob("*.mapping"): + try: + with open(mapping_file, "r") as f: + original_path = f.read().strip() + + # If this is a file in a subdirectory, fetch the whole directory + if "/" in original_path: + dir_path = original_path.split("/")[0] + + try: + # Fetch directory index + dir_files = fetcher.fetch_index(remote_path, dir_path) + + for file_name in dir_files: + full_file_path = f"{dir_path}/{file_name}" + storage_path = self.config.get_storage_path(remote_name, full_file_path) + + try: + fetcher.fetch_file(remote_path, full_file_path, storage_path) + if full_file_path not in fetched_files[remote_name]: + fetched_files[remote_name].append(full_file_path) + except Exception as e: + pass # Silently skip failed files + + except Exception as e: + # Directory might not have an index, skip silently + pass + + except Exception as e: + # Skip invalid mapping files + pass + + except Exception as e: + pass # Silently skip failed registries + + return fetched_files + + def pull_file(self, file_path: str) -> bool: + """ + Pull a specific file from storage to working copy. + + Args: + file_path: Path of the file to pull (can be full path or just filename) + + Returns: + True if file was updated, False if already up-to-date + + Raises: + DfdrError: If file not found in any remote storage + """ + # Determine working copy path - preserve directory structure if it's a full path + if "/" in file_path: + working_copy_path = self.config.project_root / file_path + else: + # For single files, use just the filename + working_copy_path = self.config.project_root / file_path + + # Find the file in storage (could be stored with full path) + storage_path = None + for remote_name in self.config.list_remotes(): + # Try the exact path first + candidate_path = self.config.get_storage_path(remote_name, file_path) + if candidate_path.exists(): + storage_path = candidate_path + break + # Also try looking for files that match the filename + remote_storage = self.config.storage_dir / remote_name + if remote_storage.exists(): + for storage_file in remote_storage.rglob("*"): + if storage_file.is_file() and not storage_file.name.endswith('.mapping'): + # Check if this matches our target file + rel_path = str(storage_file.relative_to(remote_storage)) + if rel_path == file_path or storage_file.name == Path(file_path).name: + storage_path = storage_file + break + if storage_path: + break + + if storage_path is None: + raise FileNotFoundError(f"File '{file_path}' not found in any remote storage") + + # Check if update is needed + if working_copy_path.exists(): + storage_checksum = calculate_md5(storage_path) + working_checksum = calculate_md5(working_copy_path) + + if working_checksum == storage_checksum: + return False # Already up-to-date + + # Copy from storage to working copy + working_copy_path.parent.mkdir(parents=True, exist_ok=True) + with open(storage_path, "rb") as src, open(working_copy_path, "wb") as dst: + dst.write(src.read()) + + # Update checksum, preserving registry and path info + checksum = calculate_md5(working_copy_path) + # Get original info from storage path + storage_rel_path = storage_path.relative_to(self.config.storage_dir) + registry_name = storage_rel_path.parts[0] # First part is registry name + original_path = str(Path(*storage_rel_path.parts[1:])) # Rest is the original path + save_checksum(working_copy_path, checksum, registry_name, original_path) + + return True + + def pull_all(self) -> List[str]: + """ + Pull all tracked files from storage to working copy. + Only pulls files that have corresponding .dfdr checksum files (i.e., were explicitly added). + + Returns: + List of files that were updated + """ + updated_files = [] + + # Only pull files that have .dfdr checksum files (explicitly tracked) + for dfdr_file in self.config.project_root.rglob("*.dfdr"): + data_file = Path(str(dfdr_file)[:-5]) # Remove .dfdr extension + + # Only process if the corresponding data file is actually a file, not a directory + if not data_file.is_file(): + continue + + # Get the relative path from project root for proper tracking + rel_path = str(data_file.relative_to(self.config.project_root)) + + try: + if self.pull_file(rel_path): + updated_files.append(rel_path) + except Exception as e: + pass # Silently skip failed files + + return updated_files + + def get_status(self) -> Dict[str, str]: + """ + Get status of all tracked files. + Only shows files that were explicitly added (have .dfdr checksum files). + + Returns: + Dictionary mapping file paths to status: + - "up-to-date": File matches storage + - "modified": File differs from storage + - "missing": File was tracked but is missing from working copy + - "untracked": File exists in working copy but not properly tracked + """ + status = {} + + # Only check files that have .dfdr checksum files (explicitly tracked) + for dfdr_file in self.config.project_root.rglob("*.dfdr"): + data_file = Path(str(dfdr_file)[:-5]) # Remove .dfdr extension + + # Only include if it's actually a file, not a directory + if not data_file.is_file(): + continue + + # Get the relative path from project root for proper tracking + rel_path = str(data_file.relative_to(self.config.project_root)) + + # Find corresponding file in storage + storage_path = None + for remote_name in self.config.list_remotes(): + remote_storage = self.config.storage_dir / remote_name + if remote_storage.exists(): + for storage_file in remote_storage.rglob("*"): + if storage_file.is_file() and not storage_file.name.endswith('.mapping'): + # Check if this matches our target file (by relative path or filename) + storage_rel_path = str(storage_file.relative_to(remote_storage)) + if storage_rel_path == rel_path or storage_file.name == data_file.name: + storage_path = storage_file + break + if storage_path: + break + + if storage_path is None: + status[rel_path] = "untracked" + elif not data_file.exists(): + status[rel_path] = "missing" + else: + # Check if checksum matches storage + storage_checksum = calculate_md5(storage_path) + working_checksum = calculate_md5(data_file) + + if working_checksum == storage_checksum: + status[rel_path] = "up-to-date" + else: + status[rel_path] = "modified" + + return status + + def push_file(self, file_path: str) -> bool: + """ + Push changes in a file back to its origin data registry. + + Args: + file_path: Path of the file to push (relative to project root) + + Returns: + True if file was pushed, False if no changes were needed + + Raises: + DfdrError: If file not found or not tracked + """ + working_copy_path = self.config.project_root / file_path + + if not working_copy_path.exists(): + raise FileNotFoundError(f"File '{file_path}' not found in working copy") + + # Load checksum info + checksum_info = load_checksum_info(working_copy_path) + if checksum_info is None: + raise DfdrError(f"File '{file_path}' is not tracked") + + remote_name = checksum_info["registry_name"] + original_path = checksum_info["original_path"] + + # Get remote path and storage path + remote_path = self.config.get_remote_path(remote_name) + storage_path = self.config.get_storage_path(remote_name, original_path) + + # Calculate current checksum + current_checksum = calculate_md5(working_copy_path) + + if current_checksum == checksum_info["checksum"]: + # Even if no changes, ensure storage is in sync + if storage_path.exists(): + storage_checksum = calculate_md5(storage_path) + if storage_checksum != current_checksum: + # Update storage to match working copy + storage_path.parent.mkdir(parents=True, exist_ok=True) + with open(working_copy_path, "rb") as src, open(storage_path, "wb") as dst: + dst.write(src.read()) + return False # No changes to push to remote + + # Copy file to remote + remote_file_path = remote_path / original_path + remote_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(working_copy_path, "rb") as src, open(remote_file_path, "wb") as dst: + dst.write(src.read()) + + # Update storage copy + storage_path.parent.mkdir(parents=True, exist_ok=True) + with open(working_copy_path, "rb") as src, open(storage_path, "wb") as dst: + dst.write(src.read()) + + # Update checksum info + save_checksum(working_copy_path, current_checksum, remote_name, original_path) + + return True + + def get_file_info(self, file_path: str) -> Dict[str, str]: + """ + Get origin information for a tracked file. + + Args: + file_path: Path of the file (relative to project root) + + Returns: + Dictionary containing registry_name and original_path + + Raises: + DfdrError: If file is not tracked + """ + working_copy_path = self.config.project_root / file_path + + checksum_info = load_checksum_info(working_copy_path) + if checksum_info is None: + raise DfdrError(f"File '{file_path}' is not tracked") + + return { + "registry_name": checksum_info["registry_name"], + "original_path": checksum_info["original_path"] + } + + def update_file_destination(self, file_path: str, new_destination: str) -> None: + """ + Update the destination of a tracked file. + + Args: + file_path: Current path of the file (relative to project root) + new_destination: New destination path for the file (relative to project root) + + Raises: + DfdrError: If file is not tracked or new destination is invalid + """ + working_copy_path = self.config.project_root / file_path + new_path = self.config.project_root / new_destination + + if not working_copy_path.exists(): + raise FileNotFoundError(f"File '{file_path}' not found in working copy") + + checksum_info = load_checksum_info(working_copy_path) + if checksum_info is None: + raise DfdrError(f"File '{file_path}' is not tracked") + + # Move the file to the new destination + new_path.parent.mkdir(parents=True, exist_ok=True) + working_copy_path.rename(new_path) + + # Update the checksum file + checksum = calculate_md5(new_path) + save_checksum(new_path, checksum, checksum_info["registry_name"], checksum_info["original_path"]) + + # Remove the old checksum file + (working_copy_path.parent / f"{working_copy_path.name}.dfdr").unlink() diff --git a/essai/analytics/metrics.csv b/essai/analytics/metrics.csv new file mode 100644 index 0000000..6ca2b4c --- /dev/null +++ b/essai/analytics/metrics.csv @@ -0,0 +1,3 @@ +date,page_views,unique_visitors,bounce_rate,avg_session_duration,conversion_rate +2025-05-01,1250,890,0.42,185.5,0.034 +2025-05-04,980,720,0.45,165.3,0.029 diff --git a/essai/analytics/settings.yaml b/essai/analytics/settings.yaml new file mode 100644 index 0000000..0ca3ff7 --- /dev/null +++ b/essai/analytics/settings.yaml @@ -0,0 +1,53 @@ +# Analytics dashboard configuration +dashboard: + title: "Website Analytics Dashboard" + refresh_interval: 300 # seconds + timezone: "UTC" + +# Data sources +data_sources: + - name: "web_analytics" + type: "google_analytics" + enabled: true + - name: "user_tracking" + type: "internal" + enabled: true + - name: "conversion_tracking" + type: "mixpanel" + enabled: false + +# Chart configurations +charts: + page_views: + type: "line" + time_range: "7d" + color: "#3498db" + bounce_rate: + type: "gauge" + threshold: 0.5 + color: "#e74c3c" + conversion_funnel: + type: "funnel" + steps: + - "landing_page" + - "product_view" + - "add_to_cart" + - "checkout" + - "purchase" + +# Alerts +alerts: + high_bounce_rate: + threshold: 0.6 + enabled: true + notification_email: "admin@example.com" + low_conversion: + threshold: 0.02 + enabled: true + notification_email: "marketing@example.com" + +# Export settings +export: + formats: ["csv", "json", "pdf"] + schedule: "daily" + recipients: ["analytics@example.com"] diff --git a/essai/analytics/users.json b/essai/analytics/users.json new file mode 100644 index 0000000..0405b75 --- /dev/null +++ b/essai/analytics/users.json @@ -0,0 +1,60 @@ +{ + "users": [ + { + "id": "u001", + "username": "alice_johnson", + "email": "alice@example.com", + "registration_date": "2024-03-15", + "last_login": "2025-05-06T10:30:00Z", + "profile": { + "first_name": "Alice", + "last_name": "Johnson", + "age": 28, + "location": "New York, USA" + }, + "preferences": { + "theme": "dark", + "notifications": true, + "language": "en" + } + }, + { + "id": "u002", + "username": "bob_smith", + "email": "bob@example.com", + "registration_date": "2024-01-22", + "last_login": "2025-05-05T14:45:00Z", + "profile": { + "first_name": "Bob", + "last_name": "Smith", + "age": 35, + "location": "London, UK" + }, + "preferences": { + "theme": "light", + "notifications": false, + "language": "en" + } + }, + { + "id": "u003", + "username": "maria_garcia", + "email": "maria@example.com", + "registration_date": "2024-07-08", + "last_login": "2025-05-06T09:15:00Z", + "profile": { + "first_name": "Maria", + "last_name": "Garcia", + "age": 31, + "location": "Barcelona, Spain" + }, + "preferences": { + "theme": "dark", + "notifications": true, + "language": "es" + } + } + ], + "total_users": 3, + "active_users_last_week": 2 +} diff --git a/essai/sample.json b/essai/sample.json new file mode 100644 index 0000000..53c2aed --- /dev/null +++ b/essai/sample.json @@ -0,0 +1,16 @@ +{ + "name": "Sample JSON Data", + "description": "This is a sample JSON file for the static file server", + "items": [ + { + "id": 3, + "name": "Item 4", + "value": 30.25 + } + ], + "metadata": { + "created": "2025-05-12", + "version": "1.0", + "author": "Flask Static Server" + } +} diff --git a/examples/basic_usage.py b/examples/basic_usage.py new file mode 100644 index 0000000..6a52f34 --- /dev/null +++ b/examples/basic_usage.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +""" +Basic usage example for dfdr. + +This example demonstrates how to use dfdr programmatically. +""" + +from pathlib import Path +import tempfile + +from dfdr.config import Config +from dfdr.storage import Storage + + +def main(): + """Demonstrate basic dfdr usage.""" + + # Create a temporary directory for this example + with tempfile.TemporaryDirectory() as tmpdir: + print(f"Working in temporary directory: {tmpdir}") + + # Initialize configuration + config = Config(Path(tmpdir)) + print("✓ Initialized dfdr configuration") + + # Add a remote (this would be a real URL in practice) + config.add_remote("example", "https://data.example.com/") + print("✓ Added remote 'example'") + + # List remotes + remotes = config.list_remotes() + print(f"✓ Configured remotes: {list(remotes.keys())}") + + # Initialize storage + storage = Storage(config) + print("✓ Initialized storage manager") + + # In a real scenario, you would: + # 1. storage.add_file("example", "datasets/sales.csv") + # 2. storage.fetch_all() + # 3. storage.pull_all() + # 4. status = storage.get_status() + + print("\n📁 Directory structure created:") + for path in sorted(Path(tmpdir).rglob("*")): + if path.is_file(): + rel_path = path.relative_to(tmpdir) + print(f" {rel_path}") + + +if __name__ == "__main__": + main() diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..8cbd569 --- /dev/null +++ b/install.sh @@ -0,0 +1 @@ +pip install -e . diff --git a/licence.txt b/licence.txt new file mode 100644 index 0000000..9d837fc --- /dev/null +++ b/licence.txt @@ -0,0 +1,19 @@ +Copyright (c) 2025 WhirlingAI (contact@whirlingai.com) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/readme.rst b/readme.rst new file mode 100644 index 0000000..6414e9a --- /dev/null +++ b/readme.rst @@ -0,0 +1,165 @@ + +A Python command-line tool for managing remote data registries, inspired by DVC and Git but focused specifically on data registry functionality. + +Overview +-------- + +dfdr allows you to: + +- Declare remote data sources (web servers serving CSV, JSON, YAML, TXT files over HTTP) +- Add specific files from data registries to your working copy +- Mirror data locally for efficient access +- Track file changes with checksums +- Keep your working copy synchronized with remote registries +DefDer (Data Registry Definition) - dfdr + +A Python command-line tool for managing local data registries, inspired by DVC and Git but focused specifically on data registry functionality. + +Overview +-------- + +dfdr allows you to: + +- Declare local data sources (folders containing CSV, JSON, YAML, TXT files) +- Add specific files from data registries to your working copy +- Mirror data locally for efficient access +- Track file changes with checksums +- Keep your working copy synchronized with local registries +======================================== + +A Python command-line tool for managing remote data registries, inspired by DVC and Git but focused specifically on data registry functionality. + +Overview +-------- + +dfdr allows you to: + +- Declare remote data sources (web servers serving CSV, JSON, YAML, TXT files over HTTP) +- Add specific files from data registries to your working copy +- Mirror data locally for efficient access +- Track file changes with checksums +- Keep your working copy synchronized with remote registries + +Installation +------------ + +.. code-block:: bash + + pip install dfdr + +Or install from source: + +.. code-block:: bash + + git clone https://defder.fr/dfdr.git + cd dfdr + pip install -e . + +Quick Start +----------- + +First, run + +.. code-block:: bash + + dfdr init + + +1. Add a local data registry: + +.. code-block:: bash + + dfdr registry add myregistry /path/to/local/data/folder + +2. Add files from the registry to your working copy: + +.. code-block:: bash + + dfdr add myregistry:datasets/sales.csv + dfdr add myregistry:models/config.json + +3. Update your local cache: + +.. code-block:: bash + + dfdr fetch + +4. Check the status of your files: + +.. code-block:: bash + + dfdr status + +5. Update your working copy: + +.. code-block:: bash + + dfdr pull + +Commands +-------- + +Registry Management +~~~~~~~~~~~~~~~~~~~ + +``dfdr registry add `` + Add a local data registry + +``dfdr registry list`` + List all configured registries + +``dfdr registry remove `` + Remove a local data registry + +Data Management +~~~~~~~~~~~~~~~ + +``dfdr add :`` + Add a specific file from a registry to your working copy + +``dfdr fetch`` + Mirror all data to local ``.dfdr`` storage from all remotes + +``dfdr pull [file_path]`` + Update working copy from cache (all files or specific file) + +``dfdr status`` + Show sync status of files + +Data Registry Structure +----------------------- + +Local data registries should follow this structure: + +- Files are stored in local directories +- No ``index.json`` file is required; the tool will automatically discover files in the directory + +For example: + +.. code-block:: text + + /path/to/registry/ + ├── datasets/ + │ ├── sales.csv + │ └── customers.json + └── models/ + └── config.yaml + +Local Storage +------------- + +dfdr creates a ``.dfdr`` directory in your project containing: + +- ``config.json`` - Local registry configuration +- ``storage/`` - Local mirror of registry data +- ``*.dfdr`` files - MD5 checksums for each data file (e.g., ``sales.csv.dfdr``) + +License +------- + +MIT License + +Contributing +------------ + +Contributions are welcome! Please feel free to submit a Pull Request. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2297bee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# Core dependencies +httpx>=0.24.0 +click>=8.0.0 +rich>=13.0.0 +pyyaml + +# Development dependencies (optional) +#pytest>=7.0.0 +#pytest-asyncio>=0.21.0 +#pytest-cov>=4.0.0 diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..3bbae07 --- /dev/null +++ b/setup.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +""" +Setup script for DefDer (Data Registry Definition) - dfdr +""" + +from setuptools import setup, find_packages + +with open("README.md", "r", encoding="utf-8") as fh: + long_description = fh.read() + +with open("requirements.txt", "r", encoding="utf-8") as fh: + requirements = [line.strip() for line in fh if line.strip() and not line.startswith("#")] + +setup( + name="dfdr", + version="0.1.0", + author="DefDer Team", + author_email="contact@defder.fr", + description="Data Registry Definition tool for managing remote data sources", + long_description=long_description, + long_description_content_type="text/x-rst", + url="https://github.com/defder/dfdr", + packages=find_packages(), + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Topic :: Software Development :: Libraries :: Python Modules", + ], + python_requires=">=3.8", + install_requires=requirements, + entry_points={ + "console_scripts": [ + "dfdr=dfdr.cli:main", + ], + }, + include_package_data=True, + zip_safe=False, +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..5081467 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Tests for dfdr. +""" diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..805358f --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,204 @@ +""" +Basic tests for dfdr functionality. +""" + +import pytest +from pathlib import Path +import tempfile +import shutil +import os + +from dfdr.config import Config +from dfdr.checksum import calculate_md5, save_checksum, load_checksum, load_checksum_info +from dfdr.exceptions import ConfigError, ChecksumError +from dfdr.fetcher import Fetcher +from dfdr.storage import Storage + + +class TestConfig: + """Test configuration management.""" + + def test_config_initialization(self): + """Test that config initializes correctly.""" + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(Path(tmpdir)) + assert config.project_root == Path(tmpdir) + assert config.dfdr_dir.exists() + assert config.storage_dir.exists() + + def test_add_remote(self): + """Test adding a local registry.""" + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(Path(tmpdir)) + registry_path = Path(tmpdir) / "test_registry" + registry_path.mkdir() + config.add_remote("test", str(registry_path)) + + remotes = config.list_remotes() + assert "test" in remotes + assert remotes["test"] == registry_path + + def test_duplicate_remote(self): + """Test that adding duplicate local registry raises error.""" + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(Path(tmpdir)) + registry_path = Path(tmpdir) / "test_registry" + registry_path.mkdir() + config.add_remote("test", str(registry_path)) + + with pytest.raises(ConfigError): + config.add_remote("test", str(Path(tmpdir) / "other_registry")) + + def test_remove_remote(self): + """Test removing a local registry.""" + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(Path(tmpdir)) + registry_path = Path(tmpdir) / "test_registry" + registry_path.mkdir() + config.add_remote("test", str(registry_path)) + config.remove_remote("test") + + remotes = config.list_remotes() + assert "test" not in remotes + + def test_remove_nonexistent_remote(self): + """Test that removing nonexistent remote raises error.""" + with tempfile.TemporaryDirectory() as tmpdir: + config = Config(Path(tmpdir)) + + with pytest.raises(ConfigError): + config.remove_remote("nonexistent") + + +class TestChecksum: + """Test checksum functionality.""" + + def test_calculate_md5(self): + """Test MD5 calculation.""" + with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: + f.write("test content") + f.flush() + + checksum = calculate_md5(Path(f.name)) + # MD5 of "test content" + expected = "9473fdd0d880a43c21b7778d34872157" + assert checksum == expected + assert len(checksum) == 32 # MD5 is 32 hex chars + assert isinstance(checksum, str) + + Path(f.name).unlink() + + def test_save_and_load_checksum(self): + """Test saving and loading checksums.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "test.txt" + test_file.write_text("test content") + + # Calculate and save checksum + checksum = calculate_md5(test_file) + save_checksum(test_file, checksum, "test_registry", "test.txt") + + # Load and verify + loaded_info = load_checksum_info(test_file) + assert loaded_info is not None + assert loaded_info["checksum"] == checksum + assert loaded_info["registry_name"] == "test_registry" + assert loaded_info["original_path"] == "test.txt" + + # Test the simple load_checksum function + loaded_checksum = load_checksum(test_file) + assert loaded_checksum == checksum + + # Check that .dfdr file exists + dfdr_file = Path(f"{test_file}.dfdr") + assert dfdr_file.exists() + + def test_load_nonexistent_checksum(self): + """Test loading checksum for nonexistent file.""" + with tempfile.TemporaryDirectory() as tmpdir: + test_file = Path(tmpdir) / "nonexistent.txt" + checksum = load_checksum(test_file) + assert checksum is None + + +class TestFetcher: + """Test Fetcher functionality.""" + + def test_fetch_index(self): + """Test fetching index from local registry.""" + with tempfile.TemporaryDirectory() as tmpdir: + registry_path = Path(tmpdir) / "test_registry" + registry_path.mkdir() + (registry_path / "file1.txt").touch() + (registry_path / "file2.txt").touch() + + fetcher = Fetcher() + files = fetcher.fetch_index(registry_path) + + assert set(files) == {"file1.txt", "file2.txt"} + + def test_fetch_file(self): + """Test fetching a file from local registry.""" + with tempfile.TemporaryDirectory() as tmpdir: + registry_path = Path(tmpdir) / "test_registry" + registry_path.mkdir() + source_file = registry_path / "test_file.txt" + source_file.write_text("test content") + + output_path = Path(tmpdir) / "output.txt" + + fetcher = Fetcher() + fetcher.fetch_file(registry_path, "test_file.txt", output_path) + + assert output_path.exists() + assert output_path.read_text() == "test content" + + +class TestStorage: + """Test Storage functionality.""" + + def test_add_file(self): + """Test adding a file from local registry to working copy.""" + with tempfile.TemporaryDirectory() as tmpdir: + project_root = Path(tmpdir) / "project" + project_root.mkdir() + config = Config(project_root) + + registry_path = Path(tmpdir) / "test_registry" + registry_path.mkdir() + source_file = registry_path / "test_file.txt" + source_file.write_text("test content") + + config.add_remote("test", str(registry_path)) + + storage = Storage(config) + storage.add_file("test", "test_file.txt") + + working_copy_file = project_root / "test_file.txt" + assert working_copy_file.exists() + assert working_copy_file.read_text() == "test content" + + def test_fetch_all(self): + """Test fetching all files from local registries.""" + with tempfile.TemporaryDirectory() as tmpdir: + project_root = Path(tmpdir) / "project" + project_root.mkdir() + config = Config(project_root) + + registry1_path = Path(tmpdir) / "registry1" + registry1_path.mkdir() + (registry1_path / "file1.txt").write_text("content1") + + registry2_path = Path(tmpdir) / "registry2" + registry2_path.mkdir() + (registry2_path / "file2.txt").write_text("content2") + + config.add_remote("reg1", str(registry1_path)) + config.add_remote("reg2", str(registry2_path)) + + storage = Storage(config) + fetched_files = storage.fetch_all() + + assert set(fetched_files.keys()) == {"reg1", "reg2"} + assert set(fetched_files["reg1"]) == {"file1.txt"} + assert set(fetched_files["reg2"]) == {"file2.txt"} diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000..9bed8e4 --- /dev/null +++ b/todo.txt @@ -0,0 +1,12 @@ +Todo +-------- + +- clean the dependencies + dans le requirements: + +# Core dependencies +httpx>=0.24.0 -> NON + + +- add a **real** development and tests mode +