diff --git a/README.md b/README.md index c659f13..f8c4c02 100644 --- a/README.md +++ b/README.md @@ -141,14 +141,14 @@ To bypass the configuration file and specify a vault to use at runtime use the ` ### Making bulk edits -Bulk edits are supported by importing a CSV file containing the following columns +Bulk edits are supported by importing a CSV file containing the following columns. Column headers must be lowercase. -1. `Path` - Path to note relative to the vault root folder -2. `Type` - Type of metadata. One of `frontmatter`, `inline_metadata`, or `tag` -3. `Key` - The key to add (leave blank for a tag) -4. `Value` - the value to add to the key +1. `path` - Path to note relative to the vault root folder +2. `type` - Type of metadata. One of `frontmatter`, `inline_metadata`, or `tag` +3. `key` - The key to add (leave blank for a tag) +4. `value` - the value to add to the key -Notes which match a Path in the file will be updated to contain ONLY the information in the CSV file. Notes which do not match a path will be left untouched. The example CSV below will remove any frontmatter, inline metadata, or tags within with `vault/folder 01/note1.md` and then add the specified metadata. +An example valid CSV file is ```csv path,type,key,value @@ -160,6 +160,13 @@ folder 1/note1.md,tag,,tag1 folder 1/note1.md,tag,,tag2 ``` +How bulk imports work: + +- Only notes which match the path in the CSV file are updated +- Effected notes will have ALL of their metadata changed to reflect the values in the CSV file +- Existing metadata in an effected note will be rewritten. This may result in it's location and/or formatting within the note being changed +- inline tags ignore any value added to the `key` column + You can export all your notes with their associated metadata in this format from the "Export Metadata" section of the script to be used as a template for your bulk changes. # Contributing diff --git a/src/obsidian_metadata/_utils/__init__.py b/src/obsidian_metadata/_utils/__init__.py index d609985..9d5e7be 100644 --- a/src/obsidian_metadata/_utils/__init__.py +++ b/src/obsidian_metadata/_utils/__init__.py @@ -6,10 +6,12 @@ from obsidian_metadata._utils.utilities import ( clean_dictionary, clear_screen, dict_contains, + dict_keys_to_lower, dict_values_to_lists_strings, docstring_parameter, merge_dictionaries, remove_markdown_sections, + validate_csv_bulk_imports, version_callback, ) @@ -18,11 +20,12 @@ __all__ = [ "clean_dictionary", "clear_screen", "dict_contains", + "dict_keys_to_lower", "dict_values_to_lists_strings", "docstring_parameter", "LoggerManager", "merge_dictionaries", "remove_markdown_sections", - "vault_validation", + "validate_csv_bulk_imports", "version_callback", ] diff --git a/src/obsidian_metadata/_utils/utilities.py b/src/obsidian_metadata/_utils/utilities.py index 35a2889..af7547e 100644 --- a/src/obsidian_metadata/_utils/utilities.py +++ b/src/obsidian_metadata/_utils/utilities.py @@ -1,11 +1,15 @@ """Utility functions.""" +import csv import re from os import name, system +from pathlib import Path from typing import Any import typer from obsidian_metadata.__version__ import __version__ +from obsidian_metadata._utils import alerts +from obsidian_metadata._utils.alerts import logger as log from obsidian_metadata._utils.console import console @@ -63,6 +67,18 @@ def dict_contains( return key in dictionary and value in dictionary[key] +def dict_keys_to_lower(dictionary: dict) -> dict: + """Convert all keys in a dictionary to lowercase. + + Args: + dictionary (dict): Dictionary to convert + + Returns: + dict: Dictionary with all keys converted to lowercase + """ + return {key.lower(): value for key, value in dictionary.items()} + + def dict_values_to_lists_strings( dictionary: dict, strip_null_values: bool = False, @@ -182,6 +198,55 @@ def remove_markdown_sections( return text +def validate_csv_bulk_imports(csv_path: Path, note_paths: list) -> dict[str, list[dict[str, str]]]: + """Validate the bulk import CSV file. + + Args: + csv_path (dict): Dictionary to validate + note_paths (list): List of paths to all notes in vault + + Returns: + dict: Validated dictionary + """ + csv_dict: dict[str, Any] = {} + with csv_path.expanduser().open("r") as csv_file: + csv_reader = csv.DictReader(csv_file, delimiter=",") + row_num = 0 + for row in csv_reader: + if row_num == 0: + if "path" not in row: + raise typer.BadParameter("Missing 'path' column in CSV file") + if "type" not in row: + raise typer.BadParameter("Missing 'type' column in CSV file") + if "key" not in row: + raise typer.BadParameter("Missing 'key' column in CSV file") + if "value" not in row: + raise typer.BadParameter("Missing 'value' column in CSV file") + row_num += 1 + + if row["path"] not in csv_dict: + csv_dict[row["path"]] = [] + + csv_dict[row["path"]].append( + {"type": row["type"], "key": row["key"], "value": row["value"]} + ) + + if row_num == 0 or row_num == 1: + raise typer.BadParameter("Empty CSV file") + + paths_to_remove = [x for x in csv_dict if x not in note_paths] + + for _path in paths_to_remove: + alerts.warning(f"'{_path}' does not exist in vault. Skipping...") + del csv_dict[_path] + + if len(csv_dict) == 0: + log.error("No paths in the CSV file matched paths in the vault") + raise typer.Exit(1) + + return csv_dict + + def version_callback(value: bool) -> None: """Print version and exit.""" if value: diff --git a/src/obsidian_metadata/models/application.py b/src/obsidian_metadata/models/application.py index 1167595..ef6cf56 100644 --- a/src/obsidian_metadata/models/application.py +++ b/src/obsidian_metadata/models/application.py @@ -1,7 +1,6 @@ """Questions for the cli.""" -import csv from pathlib import Path from typing import Any @@ -11,7 +10,7 @@ from rich import box from rich.table import Table from obsidian_metadata._config import VaultConfig -from obsidian_metadata._utils import alerts +from obsidian_metadata._utils import alerts, validate_csv_bulk_imports from obsidian_metadata._utils.console import console from obsidian_metadata.models import InsertLocation, Vault, VaultFilter from obsidian_metadata.models.enums import MetadataType @@ -301,18 +300,12 @@ class Application: alerts.error("File must be a CSV file") return - csv_dict: dict[str, Any] = {} - with csv_path.open("r") as csv_file: - csv_reader = csv.DictReader(csv_file, delimiter=",") - for row in csv_reader: - if row["path"] not in csv_dict: - csv_dict[row["path"]] = [] + note_paths = [ + str(n.note_path.relative_to(self.vault.vault_path)) for n in self.vault.all_notes + ] - csv_dict[row["path"]].append( - {"type": row["type"], "key": row["key"], "value": row["value"]} - ) - - num_changed = self.vault.update_from_dict(csv_dict) + dict_from_csv = validate_csv_bulk_imports(csv_path, note_paths) + num_changed = self.vault.update_from_dict(dict_from_csv) if num_changed == 0: alerts.warning("No notes were changed") diff --git a/src/obsidian_metadata/models/vault.py b/src/obsidian_metadata/models/vault.py index 49d21e1..87a6529 100644 --- a/src/obsidian_metadata/models/vault.py +++ b/src/obsidian_metadata/models/vault.py @@ -572,7 +572,7 @@ class Vault: for _note in self.all_notes: path = _note.note_path.relative_to(self.vault_path) if str(path) in dictionary: - log.debug(f"Updating metadata for {path}") + log.info(f"Updating metadata for '{path}'") num_changed += 1 _note.delete_all_metadata() for row in dictionary[str(path)]: @@ -590,7 +590,6 @@ class Vault: ) if row["type"].lower() == "tag" or row["type"].lower() == "tags": - console.print(f"Adding tag {row['value']}") _note.add_metadata( area=MetadataType.TAGS, value=row["value"], diff --git a/tests/notes_test.py b/tests/notes_test.py index 2755264..58700e0 100644 --- a/tests/notes_test.py +++ b/tests/notes_test.py @@ -264,7 +264,7 @@ def test_commit_1(sample_note, tmp_path) -> None: assert "Heading 1" not in note2.file_content -def test_commit_2(sample_note, tmp_path) -> None: +def test_commit_2(sample_note) -> None: """Test commit() method. GIVEN a note object with commit() called diff --git a/tests/utilities_test.py b/tests/utilities_test.py index d487c0f..a2b95fd 100644 --- a/tests/utilities_test.py +++ b/tests/utilities_test.py @@ -1,13 +1,18 @@ # type: ignore """Test the utilities module.""" +import pytest +import typer from obsidian_metadata._utils import ( clean_dictionary, dict_contains, + dict_keys_to_lower, dict_values_to_lists_strings, remove_markdown_sections, + validate_csv_bulk_imports, ) +from tests.helpers import Regex, remove_ansi def test_dict_contains() -> None: @@ -25,6 +30,17 @@ def test_dict_contains() -> None: assert dict_contains(d, r"key\d", "value5", is_regex=True) is True +def test_dict_keys_to_lower() -> None: + """Test the dict_keys_to_lower() function. + + GIVEN a dictionary with mixed case keys + WHEN the dict_keys_to_lower() function is called + THEN the dictionary keys should be converted to lowercase + """ + test_dict = {"Key1": "Value1", "KEY2": "Value2", "key3": "Value3"} + assert dict_keys_to_lower(test_dict) == {"key1": "Value1", "key2": "Value2", "key3": "Value3"} + + def test_dict_values_to_lists_strings(): """Test converting dictionary values to lists of strings.""" dictionary = { @@ -106,3 +122,125 @@ def test_clean_dictionary(): new_dict = clean_dictionary(dictionary) assert new_dict == {"key": ["value", "value2", "value3"]} + + +def test_validate_csv_bulk_imports_1(tmp_path): + """Test the validate_csv_bulk_imports function. + + GIVEN a csv file missing the `path` column + WHEN the validate_csv_bulk_imports function is called + THEN an exception should be raised + """ + csv_path = tmp_path / "test.csv" + csv_content = """\ +PATH,type,key,value +note1.md,type,key,value""" + csv_path.write_text(csv_content) + + with pytest.raises(typer.BadParameter): + validate_csv_bulk_imports(csv_path=csv_path, note_paths=[]) + + +def test_validate_csv_bulk_imports_2(tmp_path): + """Test the validate_csv_bulk_imports function. + + GIVEN a csv file missing the `type` column + WHEN the validate_csv_bulk_imports function is called + THEN an exception should be raised + """ + csv_path = tmp_path / "test.csv" + csv_content = """\ +path,Type,key,value +note1.md,type,key,value""" + csv_path.write_text(csv_content) + + with pytest.raises(typer.BadParameter): + validate_csv_bulk_imports(csv_path=csv_path, note_paths=[]) + + +def test_validate_csv_bulk_imports_3(tmp_path): + """Test the validate_csv_bulk_imports function. + + GIVEN a csv file missing the `key` column + WHEN the validate_csv_bulk_imports function is called + THEN an exception should be raised + """ + csv_path = tmp_path / "test.csv" + csv_content = """\ +path,type,value +note1.md,type,key,value""" + csv_path.write_text(csv_content) + + with pytest.raises(typer.BadParameter): + validate_csv_bulk_imports(csv_path=csv_path, note_paths=[]) + + +def test_validate_csv_bulk_imports_4(tmp_path): + """Test the validate_csv_bulk_imports function. + + GIVEN a csv file missing the `value` column + WHEN the validate_csv_bulk_imports function is called + THEN an exception should be raised + """ + csv_path = tmp_path / "test.csv" + csv_content = """\ +path,type,key,values +note1.md,type,key,value""" + csv_path.write_text(csv_content) + + with pytest.raises(typer.BadParameter): + validate_csv_bulk_imports(csv_path=csv_path, note_paths=[]) + + +def test_validate_csv_bulk_imports_5(tmp_path): + """Test the validate_csv_bulk_imports function. + + GIVEN a csv file with only headers + WHEN the validate_csv_bulk_imports function is called + THEN an exception should be raised + """ + csv_path = tmp_path / "test.csv" + csv_content = "path,type,key,value" + csv_path.write_text(csv_content) + + with pytest.raises(typer.BadParameter): + validate_csv_bulk_imports(csv_path=csv_path, note_paths=[]) + + +def test_validate_csv_bulk_imports_6(tmp_path, capsys): + """Test the validate_csv_bulk_imports function. + + GIVEN a valid csv file + WHEN a path is given that does not exist in the vault + THEN show the user a warning + """ + csv_path = tmp_path / "test.csv" + csv_content = """\ +path,type,key,value +note1.md,type,key,value +note2.md,type,key,value +""" + csv_path.write_text(csv_content) + + csv_dict = validate_csv_bulk_imports(csv_path=csv_path, note_paths=["note1.md"]) + captured = remove_ansi(capsys.readouterr().out) + assert "WARNING | 'note2.md' does not exist in vault." in captured + assert csv_dict == {"note1.md": [{"key": "key", "type": "type", "value": "value"}]} + + +def test_validate_csv_bulk_imports_7(tmp_path): + """Test the validate_csv_bulk_imports function. + + GIVEN a valid csv file + WHEN no paths match paths in the vault + THEN exit the program + """ + csv_path = tmp_path / "test.csv" + csv_content = """\ +path,type,key,value +note1.md,type,key,value +note2.md,type,key,value +""" + csv_path.write_text(csv_content) + with pytest.raises(typer.Exit): + validate_csv_bulk_imports(csv_path=csv_path, note_paths=[]) diff --git a/tests/vault_test.py b/tests/vault_test.py index c1e01eb..3d2beeb 100644 --- a/tests/vault_test.py +++ b/tests/vault_test.py @@ -239,7 +239,7 @@ def test_commit_changes_2(test_vault, tmp_path): assert "new_key: new_key_value" not in committed_content -def test_backup_1(test_vault, tmp_path, capsys): +def test_backup_1(test_vault, capsys): """Test the backup method. GIVEN a vault object