System

A system is a group of interconnected components or resources that work together to achieve a common goal. Belonging to an organisation, a system can take on various forms, such as a GCP Cloud instance, AWS instance, front-end, back-end, or third-party SAAS. A system usually comprises several layers of resources, such as a database server, storage devices, or computing hardware.

The system is defined in the catalogue/\<organization name\>/\<system name\>/system.yml. The system.yml file accepts all arguments defined in the System model with the expection of:

1. The key is created using the catalogue folder structure.
2. The resources dictionary is created using the catalogue folder strucutre.

Example

# system.yaml

system:
  key: system_key
  tags:
  - foo
  - bar
  name: system_foo
  description: <system description> [optional]

Models

blackline.models.catalogue.System

Bases: BlacklineModel

The System resource model.

Systems can be assigned to this resource, but it doesn't inherently point to any other resources.

Parameters:

Name	Type	Description	Default
children	Optional[dict[str, Resource]]		None
stem			required
children_stem			required
children_cls			required

Source code in BAR /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/blackline/models/catalogue.py

class System(BlacklineModel):
    """
    The System resource model.

    Systems can be assigned to this resource, but it doesn't inherently
    point to any other resources.
    """

    children: Optional[dict[str, Resource]] = Field(alias="resources")  # type: ignore[assignment]

    stem = "system"
    children_stem = "resources"
    children_cls = Resource

blackline.models.catalogue.BlacklineModel

Bases: BaseModel

The base model for all Resources.

Parameters:

Name	Type	Description	Default
key	Key	A unique key used to identify this resource.	required
tags	Optional[list[str]]	A list of tags for this resource.	None
name	Optional[str]		None
description	Optional[str]		None
children	Optional[dict[str, Type[BlacklineModel]]]	The children resources.	None
stem	str	The stem of the resource.	required
children_stem	Optional[str]		required
children_cls	Optional[type[BlacklineModel]]		required

Source code in BAR /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/blackline/models/catalogue.py

class BlacklineModel(BaseModel):
    """The base model for all Resources."""

    key: Key = Field(description="A unique key used to identify this resource.")
    tags: Optional[list[str]] = Field(description="A list of tags for this resource.")
    name: Optional[str] = name_field
    description: Optional[str] = description_field
    children: Optional[dict[str, Type[BlacklineModel]]] = Field(
        None, description="The children resources."
    )
    stem: ClassVar[str] = Field(description="The stem of the resource.")
    children_stem: ClassVar[Optional[str]] = None
    children_cls: ClassVar[Optional[type[BlacklineModel]]] = None

    class Config:
        "Config for the BlacklineModel"
        extra = "forbid"
        orm_mode = True

    def __getitem__(self, key: str) -> Type[BlacklineModel]:
        parts = key.split(".")
        key = ".".join([self.key, parts[0]])
        if self.children is None:
            raise KeyError(f"No children for {self.key}")
        model = self.children[key]

        for part in parts[1:]:
            model = model[part]  # type: ignore[index]
        return model

    @classmethod
    def parse_dir(cls, path: Path, key_prefix: Optional[str] = None):
        """
        Parse a directory of YAML files into a dictionary of Dataset objects.

        Args:
            path: The path to the directory of YAML files.
            path: Path

        Returns:
            A dictionary of Dataset objects.
        """
        key = ".".join([key_prefix, path.name]) if key_prefix is not None else path.name
        children = cls.parse_children(path=path, key_prefix=key)
        filepath = cls.find_definition_file(path=path)
        return cls.parse_yaml(path=filepath, key=key, children=children)

    @classmethod
    def parse_children(
        cls, path: Path, key_prefix: Optional[str] = None
    ) -> dict[str, Type[BlacklineModel]]:
        """
        Parse a directory of YAML files into a dictionary of Dataset objects.

        Args:
            path: The path to the directory of YAML files.
            path: Path

        Returns:
            A dictionary of Dataset objects.
        """
        children: dict[str, Type[BlacklineModel]] = {}
        if cls.children_cls is None:
            return children
        for child_path in path.iterdir():
            if child_path.is_dir():
                child = cls.children_cls.parse_dir(
                    path=child_path, key_prefix=key_prefix
                )
                children[child.key] = child
        return children

    @classmethod
    def find_definition_file(cls, path: Path) -> Path:
        file = list(path.glob(f"{cls.stem}.yml")) + list(path.glob(f"{cls.stem}.yaml"))
        file_len = len(list(file))
        if file_len == 0:
            raise FileNotFoundError(
                f"No {cls.stem} file found in directory: {path.absolute()}"
            )
        if file_len > 1:
            raise ValueError(
                f"Multiple {cls.stem} files found in directory: {path.absolute()}, only include one of resource.yaml or resource.yml"
            )
        return file[0]

    @classmethod
    def parse_yaml(
        cls,
        path: Path,
        key: str,
        children: Optional[dict[str, Type[BlacklineModel]]] = {},
    ):
        """
        Parse a yaml file into a the children_cls object.

        Args:
            path: Path location of the yaml file.
            key: Key to identify the dataset.

        Returns:
            Dataset object.
        """
        with open(path, "r") as f:
            info = yaml.safe_load(f)[cls.stem][0]
            info["key"] = key
            if cls.stem == "dataset":
                return cls.parse_obj(info)
            info[cls.children_stem] = children
            return cls.parse_obj(info)

Config

Config for the BlacklineModel

Source code in BAR /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/blackline/models/catalogue.py

class Config:
    "Config for the BlacklineModel"
    extra = "forbid"
    orm_mode = True

parse_children(path, key_prefix=None) classmethod

Parse a directory of YAML files into a dictionary of Dataset objects.

Parameters:

Name	Type	Description	Default
path	Path	The path to the directory of YAML files.	required
path	Path	Path	required

Returns:

Type	Description
dict[str, Type[BlacklineModel]]	A dictionary of Dataset objects.

Source code in /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/blackline/models/catalogue.py

@classmethod
def parse_children(
    cls, path: Path, key_prefix: Optional[str] = None
) -> dict[str, Type[BlacklineModel]]:
    """
    Parse a directory of YAML files into a dictionary of Dataset objects.

    Args:
        path: The path to the directory of YAML files.
        path: Path

    Returns:
        A dictionary of Dataset objects.
    """
    children: dict[str, Type[BlacklineModel]] = {}
    if cls.children_cls is None:
        return children
    for child_path in path.iterdir():
        if child_path.is_dir():
            child = cls.children_cls.parse_dir(
                path=child_path, key_prefix=key_prefix
            )
            children[child.key] = child
    return children

parse_dir(path, key_prefix=None) classmethod

Parse a directory of YAML files into a dictionary of Dataset objects.

Parameters:

Name	Type	Description	Default
path	Path	The path to the directory of YAML files.	required
path	Path	Path	required

Returns:

Type	Description
	A dictionary of Dataset objects.

Source code in /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/blackline/models/catalogue.py

@classmethod
def parse_dir(cls, path: Path, key_prefix: Optional[str] = None):
    """
    Parse a directory of YAML files into a dictionary of Dataset objects.

    Args:
        path: The path to the directory of YAML files.
        path: Path

    Returns:
        A dictionary of Dataset objects.
    """
    key = ".".join([key_prefix, path.name]) if key_prefix is not None else path.name
    children = cls.parse_children(path=path, key_prefix=key)
    filepath = cls.find_definition_file(path=path)
    return cls.parse_yaml(path=filepath, key=key, children=children)

parse_yaml(path, key, children={}) classmethod

Parse a yaml file into a the children_cls object.

Parameters:

Name	Type	Description	Default
path	Path	Path location of the yaml file.	required
key	str	Key to identify the dataset.	required

Returns:

Type	Description
	Dataset object.

Source code in /opt/hostedtoolcache/Python/3.10.12/x64/lib/python3.10/site-packages/blackline/models/catalogue.py

@classmethod
def parse_yaml(
    cls,
    path: Path,
    key: str,
    children: Optional[dict[str, Type[BlacklineModel]]] = {},
):
    """
    Parse a yaml file into a the children_cls object.

    Args:
        path: Path location of the yaml file.
        key: Key to identify the dataset.

    Returns:
        Dataset object.
    """
    with open(path, "r") as f:
        info = yaml.safe_load(f)[cls.stem][0]
        info["key"] = key
        if cls.stem == "dataset":
            return cls.parse_obj(info)
        info[cls.children_stem] = children
        return cls.parse_obj(info)