605 lines
20 KiB
Python
605 lines
20 KiB
Python
"""Core taxonomy data structures and operations."""
|
|
|
|
import os
|
|
import json
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from typing import Optional
|
|
|
|
from . import shlex_parser
|
|
from . import timestamp
|
|
|
|
|
|
# Custom exceptions
|
|
class TaxonomyException(Exception):
|
|
"""Base exception for taxonomy errors."""
|
|
|
|
pass
|
|
|
|
|
|
class ValidationFailed(TaxonomyException):
|
|
"""Raised when validation fails."""
|
|
|
|
pass
|
|
|
|
|
|
class InvalidEcosystem(TaxonomyException):
|
|
"""Raised when ecosystem doesn't exist."""
|
|
|
|
pass
|
|
|
|
|
|
class InvalidParentEcosystem(TaxonomyException):
|
|
"""Raised when parent ecosystem doesn't exist."""
|
|
|
|
pass
|
|
|
|
|
|
class InvalidChildEcosystem(TaxonomyException):
|
|
"""Raised when child ecosystem doesn't exist."""
|
|
|
|
pass
|
|
|
|
|
|
class InvalidSourceRepo(TaxonomyException):
|
|
"""Raised when source repo doesn't exist."""
|
|
|
|
pass
|
|
|
|
|
|
class InvalidSourceEcosystem(TaxonomyException):
|
|
"""Raised when source ecosystem doesn't exist."""
|
|
|
|
pass
|
|
|
|
|
|
class DestinationEcosystemAlreadyExists(TaxonomyException):
|
|
"""Raised when destination ecosystem already exists."""
|
|
|
|
pass
|
|
|
|
|
|
class InvalidRepo(TaxonomyException):
|
|
"""Raised when repo doesn't exist."""
|
|
|
|
pass
|
|
|
|
|
|
class EcosystemHasNoRepos(TaxonomyException):
|
|
"""Raised when ecosystem has no repos."""
|
|
|
|
pass
|
|
|
|
|
|
class ParentEcosystemHasNoChildren(TaxonomyException):
|
|
"""Raised when parent has no children."""
|
|
|
|
pass
|
|
|
|
|
|
class EcoAddRequiresOneParameter(TaxonomyException):
|
|
"""Raised when ecoadd doesn't have exactly one parameter."""
|
|
|
|
pass
|
|
|
|
|
|
class EcoConRequiresExactlyTwoParameters(TaxonomyException):
|
|
"""Raised when ecocon doesn't have exactly two parameters."""
|
|
|
|
pass
|
|
|
|
|
|
class EcoDisRequiresExactlyTwoParameters(TaxonomyException):
|
|
"""Raised when ecodis doesn't have exactly two parameters."""
|
|
|
|
pass
|
|
|
|
|
|
class EcoRemRequiresExactlyTwoParameters(TaxonomyException):
|
|
"""Raised when ecorem doesn't have exactly one parameter."""
|
|
|
|
pass
|
|
|
|
|
|
class RepAddRequiresAtLeastTwoParameters(TaxonomyException):
|
|
"""Raised when repadd doesn't have at least two parameters."""
|
|
|
|
pass
|
|
|
|
|
|
class RepMovRequiresExactlyTwoParameters(TaxonomyException):
|
|
"""Raised when repmov doesn't have exactly two parameters."""
|
|
|
|
pass
|
|
|
|
|
|
class EcoMovRequiresExactlyTwoParameters(TaxonomyException):
|
|
"""Raised when ecomov doesn't have exactly two parameters."""
|
|
|
|
pass
|
|
|
|
|
|
class RepRemRequiresExactlyTwoParameters(TaxonomyException):
|
|
"""Raised when reprem doesn't have exactly two parameters."""
|
|
|
|
pass
|
|
|
|
|
|
class UnterminatedQuote(TaxonomyException):
|
|
"""Raised when a quote is not terminated."""
|
|
|
|
pass
|
|
|
|
|
|
@dataclass
|
|
class TaxonomyError:
|
|
"""Represents an error during taxonomy processing."""
|
|
|
|
message: str
|
|
line_num: int
|
|
path: str
|
|
|
|
|
|
@dataclass
|
|
class TaxonomyStats:
|
|
"""Statistics about the taxonomy."""
|
|
|
|
migration_count: int
|
|
eco_count: int
|
|
repo_count: int
|
|
tag_count: int
|
|
eco_connections_count: int = 0
|
|
|
|
|
|
class Taxonomy:
|
|
"""Main taxonomy class that manages ecosystems, repos, and their relationships."""
|
|
|
|
def __init__(self):
|
|
"""Initialize an empty taxonomy."""
|
|
self.eco_auto_id: int = 0
|
|
self.repo_auto_id: int = 0
|
|
self.tag_auto_id: int = 0
|
|
self.migration_count: int = 0
|
|
|
|
# String to ID mappings
|
|
self.eco_ids: dict[str, int] = {}
|
|
self.repo_ids: dict[str, int] = {}
|
|
self.tag_ids: dict[str, int] = {}
|
|
|
|
# ID to string mappings
|
|
self.tag_id_to_name: dict[int, str] = {}
|
|
self.repo_id_to_url: dict[int, str] = {}
|
|
self.eco_id_to_name: dict[int, str] = {}
|
|
|
|
# Relationship mappings
|
|
self.eco_to_repos: dict[int, set[int]] = {}
|
|
self.repo_to_eco: dict[int, int] = {}
|
|
self.parent_to_children: dict[int, set[int]] = {}
|
|
self.child_to_parents: dict[int, set[int]] = {}
|
|
self.eco_repo_to_tags: dict[tuple[int, int], set[int]] = {}
|
|
|
|
# Error tracking
|
|
self.errors: list[TaxonomyError] = []
|
|
|
|
def stats(self) -> TaxonomyStats:
|
|
"""
|
|
Return statistics about the taxonomy.
|
|
|
|
Returns:
|
|
TaxonomyStats with counts of migrations, ecosystems, repos, and tags
|
|
"""
|
|
return TaxonomyStats(
|
|
migration_count=self.migration_count,
|
|
eco_count=len(self.eco_ids),
|
|
repo_count=len(self.repo_ids),
|
|
tag_count=len(self.tag_ids),
|
|
eco_connections_count=0,
|
|
)
|
|
|
|
def _add_eco(self, name: str) -> None:
|
|
"""Add a new ecosystem."""
|
|
if name not in self.eco_ids:
|
|
self.eco_auto_id += 1
|
|
self.eco_ids[name] = self.eco_auto_id
|
|
self.eco_id_to_name[self.eco_auto_id] = name
|
|
|
|
def _connect_eco(self, parent: str, child: str) -> None:
|
|
"""Connect a child ecosystem to a parent."""
|
|
if parent not in self.eco_ids:
|
|
raise InvalidParentEcosystem(f"Parent ecosystem '{parent}' does not exist")
|
|
if child not in self.eco_ids:
|
|
raise InvalidChildEcosystem(f"Child ecosystem '{child}' does not exist")
|
|
|
|
parent_id = self.eco_ids[parent]
|
|
child_id = self.eco_ids[child]
|
|
|
|
# Add to parent -> children mapping
|
|
if parent_id not in self.parent_to_children:
|
|
self.parent_to_children[parent_id] = set()
|
|
self.parent_to_children[parent_id].add(child_id)
|
|
|
|
# Add to child -> parents mapping
|
|
if child_id not in self.child_to_parents:
|
|
self.child_to_parents[child_id] = set()
|
|
self.child_to_parents[child_id].add(parent_id)
|
|
|
|
def _disconnect_eco(self, parent: str, child: str) -> None:
|
|
"""Disconnect a child ecosystem from a parent."""
|
|
if parent not in self.eco_ids:
|
|
raise InvalidParentEcosystem(f"Parent ecosystem '{parent}' does not exist")
|
|
if child not in self.eco_ids:
|
|
raise InvalidChildEcosystem(f"Child ecosystem '{child}' does not exist")
|
|
|
|
parent_id = self.eco_ids[parent]
|
|
child_id = self.eco_ids[child]
|
|
|
|
# Remove from parent -> children mapping
|
|
if parent_id not in self.parent_to_children:
|
|
raise ParentEcosystemHasNoChildren(f"Parent '{parent}' has no children")
|
|
self.parent_to_children[parent_id].discard(child_id)
|
|
|
|
# Remove from child -> parents mapping
|
|
if child_id in self.child_to_parents:
|
|
self.child_to_parents[child_id].discard(parent_id)
|
|
|
|
def _remove_eco_by_id(self, eco_id: int) -> None:
|
|
"""Remove ecosystem relationships by ID."""
|
|
# Remove from parent relationships
|
|
if eco_id in self.child_to_parents:
|
|
parent_set = self.child_to_parents[eco_id]
|
|
for parent_id in parent_set:
|
|
if parent_id in self.parent_to_children:
|
|
self.parent_to_children[parent_id].discard(eco_id)
|
|
del self.child_to_parents[eco_id]
|
|
|
|
# Remove from child relationships
|
|
if eco_id in self.parent_to_children:
|
|
del self.parent_to_children[eco_id]
|
|
|
|
def _remove_eco(self, eco_name: str) -> None:
|
|
"""Remove an ecosystem."""
|
|
if eco_name not in self.eco_ids:
|
|
raise InvalidEcosystem(f"Ecosystem '{eco_name}' does not exist")
|
|
|
|
eco_id = self.eco_ids[eco_name]
|
|
self._remove_eco_by_id(eco_id)
|
|
del self.eco_ids[eco_name]
|
|
|
|
def _add_repo(
|
|
self, eco_name: str, repo_url: str, tags: Optional[list[str]] = None
|
|
) -> None:
|
|
"""Add a repository to an ecosystem with optional tags."""
|
|
if eco_name not in self.eco_ids:
|
|
raise InvalidEcosystem(f"Ecosystem '{eco_name}' does not exist")
|
|
|
|
eco_id = self.eco_ids[eco_name]
|
|
|
|
# Add or get repo ID
|
|
if repo_url not in self.repo_ids:
|
|
self.repo_auto_id += 1
|
|
self.repo_ids[repo_url] = self.repo_auto_id
|
|
self.repo_id_to_url[self.repo_auto_id] = repo_url
|
|
|
|
repo_id = self.repo_ids[repo_url]
|
|
|
|
# Add repo to ecosystem
|
|
if eco_id not in self.eco_to_repos:
|
|
self.eco_to_repos[eco_id] = set()
|
|
self.eco_to_repos[eco_id].add(repo_id)
|
|
|
|
# Add tags if provided
|
|
if tags:
|
|
for tag in tags:
|
|
if tag not in self.tag_ids:
|
|
self.tag_auto_id += 1
|
|
self.tag_ids[tag] = self.tag_auto_id
|
|
self.tag_id_to_name[self.tag_auto_id] = tag
|
|
|
|
tag_id = self.tag_ids[tag]
|
|
key = (eco_id, repo_id)
|
|
|
|
if key not in self.eco_repo_to_tags:
|
|
self.eco_repo_to_tags[key] = set()
|
|
self.eco_repo_to_tags[key].add(tag_id)
|
|
|
|
def _move_repo(self, src: str, dst: str) -> None:
|
|
"""Move/rename a repository URL."""
|
|
if src not in self.repo_ids:
|
|
raise InvalidSourceRepo(f"Source repo '{src}' does not exist")
|
|
|
|
src_id = self.repo_ids[src]
|
|
|
|
if dst in self.repo_ids:
|
|
# Destination exists, merge them
|
|
dst_id = self.repo_ids[dst]
|
|
|
|
# Update all ecosystems that have src to use dst
|
|
for eco_id, repo_set in self.eco_to_repos.items():
|
|
if src_id in repo_set:
|
|
repo_set.discard(src_id)
|
|
repo_set.add(dst_id)
|
|
|
|
# Remove src from mappings
|
|
del self.repo_ids[src]
|
|
del self.repo_id_to_url[src_id]
|
|
else:
|
|
# Simple rename
|
|
del self.repo_ids[src]
|
|
self.repo_id_to_url[src_id] = dst
|
|
self.repo_ids[dst] = src_id
|
|
|
|
def _move_eco(self, src: str, dst: str) -> None:
|
|
"""Move/rename an ecosystem."""
|
|
if src not in self.eco_ids:
|
|
raise InvalidSourceEcosystem(f"Source ecosystem '{src}' does not exist")
|
|
if dst in self.eco_ids:
|
|
raise DestinationEcosystemAlreadyExists(
|
|
f"Destination ecosystem '{dst}' already exists"
|
|
)
|
|
|
|
src_id = self.eco_ids[src]
|
|
|
|
# Update mappings
|
|
del self.eco_ids[src]
|
|
self.eco_id_to_name[src_id] = dst
|
|
self.eco_ids[dst] = src_id
|
|
|
|
def _remove_repo_from_ecosystem(self, eco_name: str, repo_url: str) -> None:
|
|
"""Remove a repository from an ecosystem."""
|
|
if eco_name not in self.eco_ids:
|
|
raise InvalidEcosystem(f"Ecosystem '{eco_name}' does not exist")
|
|
|
|
eco_id = self.eco_ids[eco_name]
|
|
|
|
if eco_id not in self.eco_to_repos:
|
|
raise EcosystemHasNoRepos(f"Ecosystem '{eco_name}' has no repos")
|
|
|
|
if repo_url not in self.repo_ids:
|
|
raise InvalidRepo(f"Repo '{repo_url}' does not exist")
|
|
|
|
repo_id = self.repo_ids[repo_url]
|
|
self.eco_to_repos[eco_id].discard(repo_id)
|
|
|
|
# Remove tags for this eco-repo combination
|
|
key = (eco_id, repo_id)
|
|
if key in self.eco_repo_to_tags:
|
|
del self.eco_repo_to_tags[key]
|
|
|
|
# Command handlers
|
|
def _cmd_ecoadd(self, tokens: list[str]) -> None:
|
|
"""Handle ecoadd command."""
|
|
if len(tokens) != 1:
|
|
raise EcoAddRequiresOneParameter("ecoadd requires exactly one parameter")
|
|
self._add_eco(tokens[0])
|
|
|
|
def _cmd_repadd(self, tokens: list[str]) -> None:
|
|
"""Handle repadd command."""
|
|
if len(tokens) < 2:
|
|
raise RepAddRequiresAtLeastTwoParameters(
|
|
"repadd requires at least two parameters"
|
|
)
|
|
|
|
eco_name = tokens[0]
|
|
repo_url = tokens[1]
|
|
tags = tokens[2:] if len(tokens) > 2 else None
|
|
self._add_repo(eco_name, repo_url, tags)
|
|
|
|
def _cmd_ecocon(self, tokens: list[str]) -> None:
|
|
"""Handle ecocon command."""
|
|
if len(tokens) != 2:
|
|
raise EcoConRequiresExactlyTwoParameters(
|
|
"ecocon requires exactly two parameters"
|
|
)
|
|
self._connect_eco(tokens[0], tokens[1])
|
|
|
|
def _cmd_ecodis(self, tokens: list[str]) -> None:
|
|
"""Handle ecodis command."""
|
|
if len(tokens) != 2:
|
|
raise EcoDisRequiresExactlyTwoParameters(
|
|
"ecodis requires exactly two parameters"
|
|
)
|
|
self._disconnect_eco(tokens[0], tokens[1])
|
|
|
|
def _cmd_ecorem(self, tokens: list[str]) -> None:
|
|
"""Handle ecorem command."""
|
|
if len(tokens) != 1:
|
|
raise EcoRemRequiresExactlyTwoParameters(
|
|
"ecorem requires exactly one parameter"
|
|
)
|
|
self._remove_eco(tokens[0])
|
|
|
|
def _cmd_repmov(self, tokens: list[str]) -> None:
|
|
"""Handle repmov command."""
|
|
if len(tokens) != 2:
|
|
raise RepMovRequiresExactlyTwoParameters(
|
|
"repmov requires exactly two parameters"
|
|
)
|
|
self._move_repo(tokens[0], tokens[1])
|
|
|
|
def _cmd_ecomov(self, tokens: list[str]) -> None:
|
|
"""Handle ecomov command."""
|
|
if len(tokens) != 2:
|
|
raise EcoMovRequiresExactlyTwoParameters(
|
|
"ecomov requires exactly two parameters"
|
|
)
|
|
self._move_eco(tokens[0], tokens[1])
|
|
|
|
def _cmd_reprem(self, tokens: list[str]) -> None:
|
|
"""Handle reprem command."""
|
|
if len(tokens) != 2:
|
|
raise RepRemRequiresExactlyTwoParameters(
|
|
"reprem requires exactly two parameters"
|
|
)
|
|
self._remove_repo_from_ecosystem(tokens[0], tokens[1])
|
|
|
|
def _is_comment(self, line: str) -> bool:
|
|
"""Check if a line is a comment."""
|
|
stripped = line.lstrip()
|
|
return len(stripped) == 0 or stripped[0] == "#"
|
|
|
|
def _load_file(self, path: str) -> None:
|
|
"""Load and process a single migration file."""
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
content = f.read()
|
|
|
|
lines = content.split("\n")
|
|
for line_num, line in enumerate(lines, start=1):
|
|
# Skip comments and blank lines
|
|
if self._is_comment(line):
|
|
continue
|
|
|
|
# Skip lines too short to have a command
|
|
if len(line) < 6:
|
|
continue
|
|
|
|
# Parse command
|
|
keyword = line[0:6]
|
|
remainder = line[6:]
|
|
|
|
try:
|
|
# Parse tokens from remainder
|
|
tokens = shlex_parser.split(remainder)
|
|
|
|
# Execute command
|
|
if keyword == "ecoadd":
|
|
self._cmd_ecoadd(tokens)
|
|
elif keyword == "repadd":
|
|
self._cmd_repadd(tokens)
|
|
elif keyword == "ecocon":
|
|
self._cmd_ecocon(tokens)
|
|
elif keyword == "ecodis":
|
|
self._cmd_ecodis(tokens)
|
|
elif keyword == "ecorem":
|
|
self._cmd_ecorem(tokens)
|
|
elif keyword == "repmov":
|
|
self._cmd_repmov(tokens)
|
|
elif keyword == "ecomov":
|
|
self._cmd_ecomov(tokens)
|
|
elif keyword == "reprem":
|
|
self._cmd_reprem(tokens)
|
|
# Ignore unknown commands (might be comments or empty lines)
|
|
|
|
except (TaxonomyException, ValueError) as e:
|
|
error_name = type(e).__name__
|
|
self.errors.append(
|
|
TaxonomyError(message=error_name, line_num=line_num, path=path)
|
|
)
|
|
|
|
def load(self, root_dir: str, max_date: Optional[str] = None) -> None:
|
|
"""
|
|
Load and process migration files from root_dir.
|
|
|
|
Args:
|
|
root_dir: Directory containing migration files
|
|
max_date: Optional maximum date to filter migrations (YYYY-MM-DD format)
|
|
|
|
Raises:
|
|
ValidationFailed: If errors were encountered during loading
|
|
"""
|
|
# List all files in directory
|
|
try:
|
|
files = os.listdir(root_dir)
|
|
except OSError as e:
|
|
raise TaxonomyException(f"Cannot read directory '{root_dir}': {e}")
|
|
|
|
# Filter files with valid timestamps
|
|
migration_files = []
|
|
for filename in files:
|
|
file_path = os.path.join(root_dir, filename)
|
|
if os.path.isfile(file_path) and timestamp.has_valid_timestamp(filename):
|
|
# Apply date filter if provided
|
|
if max_date is None or filename < max_date:
|
|
migration_files.append(filename)
|
|
|
|
# Sort files by timestamp (first 19 characters)
|
|
migration_files.sort(key=lambda x: x[0:19])
|
|
|
|
# Process each file
|
|
for filename in migration_files:
|
|
file_path = os.path.join(root_dir, filename)
|
|
self._load_file(file_path)
|
|
self.migration_count += 1
|
|
|
|
# Check for errors
|
|
if self.errors:
|
|
self._print_errors()
|
|
raise ValidationFailed("Validation failed with errors")
|
|
|
|
def _print_errors(self) -> None:
|
|
"""Print all errors to stderr."""
|
|
for error in self.errors:
|
|
print(
|
|
f"{error.path}:{error.line_num}: error.{error.message}", file=sys.stderr
|
|
)
|
|
|
|
def _tag_strings_for_eco_repo(
|
|
self, eco_id: int, repo_id: int
|
|
) -> Optional[list[str]]:
|
|
"""Get tag strings for a specific ecosystem-repo combination."""
|
|
key = (eco_id, repo_id)
|
|
if key in self.eco_repo_to_tags:
|
|
tag_ids = self.eco_repo_to_tags[key]
|
|
return [self.tag_id_to_name[tag_id] for tag_id in tag_ids]
|
|
return None
|
|
|
|
def _emit_ecosystem_json(self, f, top: str, branch: list[str], eco_id: int) -> None:
|
|
"""Recursively emit JSON for an ecosystem and its sub-ecosystems."""
|
|
# Get repos for this ecosystem and sort them
|
|
if eco_id in self.eco_to_repos:
|
|
repo_ids = self.eco_to_repos[eco_id]
|
|
# Create tuples of (repo_id, repo_url) and sort by URL (case-insensitive)
|
|
repo_tuples = [
|
|
(repo_id, self.repo_id_to_url[repo_id]) for repo_id in repo_ids
|
|
]
|
|
repo_tuples.sort(key=lambda x: x[1].casefold())
|
|
|
|
# Emit each repo
|
|
for repo_id, repo_url in repo_tuples:
|
|
tags = self._tag_strings_for_eco_repo(eco_id, repo_id)
|
|
row = {
|
|
"eco_name": top,
|
|
"branch": branch.copy(),
|
|
"repo_url": repo_url,
|
|
"tags": tags if tags else [],
|
|
}
|
|
f.write(json.dumps(row, separators=(",", ":")) + "\n")
|
|
|
|
# Get sub-ecosystems and sort them
|
|
if eco_id in self.parent_to_children:
|
|
child_ids = self.parent_to_children[eco_id]
|
|
# Create tuples of (child_id, child_name) and sort by name (case-insensitive)
|
|
child_tuples = [
|
|
(child_id, self.eco_id_to_name[child_id]) for child_id in child_ids
|
|
]
|
|
child_tuples.sort(key=lambda x: x[1].casefold())
|
|
|
|
# Recursively emit each sub-ecosystem
|
|
for child_id, child_name in child_tuples:
|
|
branch.append(child_name)
|
|
self._emit_ecosystem_json(f, top, branch, child_id)
|
|
branch.pop()
|
|
|
|
def export_json(self, output_file: str, ecosystem: Optional[str] = None) -> None:
|
|
"""
|
|
Export taxonomy to JSON Lines format.
|
|
|
|
Args:
|
|
output_file: Path to output file
|
|
ecosystem: Optional specific ecosystem to export (exports all if None)
|
|
"""
|
|
# Collect ecosystems to export
|
|
if ecosystem:
|
|
if ecosystem not in self.eco_ids:
|
|
raise InvalidEcosystem(f"Ecosystem '{ecosystem}' does not exist")
|
|
keys_list = [(ecosystem, self.eco_ids[ecosystem])]
|
|
else:
|
|
# Get all ecosystems and sort by name (case-insensitive)
|
|
keys_list = [(name, eco_id) for name, eco_id in self.eco_ids.items()]
|
|
keys_list.sort(key=lambda x: x[0].casefold())
|
|
|
|
# Write to file
|
|
with open(output_file, "w", encoding="utf-8") as f:
|
|
for eco_name, eco_id in keys_list:
|
|
branch = []
|
|
self._emit_ecosystem_json(f, eco_name, branch, eco_id)
|