Merge pull request #802 from elementary-data/master

Merge pull request #801 from elementary-data/release/v0.7.7

Author: elongl

Dockerfile

@@ -6,7 +6,18 @@ ENV DBT_TARGET_PATH=$USR_APP_PATH/target
 WORKDIR $USR_APP_PATH
 RUN chmod 777 .
 
+RUN apt-get update \
+    && apt-get dist-upgrade -y \
+    && apt-get install -y --no-install-recommends \
+    python-dev \
+    libsasl2-dev \
+    && apt-get clean \
+    && rm -rf \
+    /var/lib/apt/lists/* \
+    /tmp/* \
+    /var/tmp/*
+
 COPY . /app
-RUN pip install --no-cache-dir "/app[postgres, snowflake, bigquery, redshift, databricks]"
+RUN pip install --no-cache-dir "/app[all]"
 
 ENTRYPOINT ["edr"]

README.md

@@ -174,6 +174,8 @@ Check out the [contributions guide](https://docs.elementary-data.com/general/con
 <a href="https://github.com/vishaalkk"><img src="https://github.com/u/13641827?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/sadahry"><img src="https://github.com/u/28292300?v=4" width="50" height="50" alt=""/></a>
 <a href="https://github.com/will-warner"><img src="https://github.com/u/110092386?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/winzee"><img src="https://github.com/u/1001577?v=4" width="50" height="50" alt=""/></a>
+<a href="https://github.com/vinooganesh"><img src="https://github.com/u/2461070?v=4" width="50" height="50" alt=""/></a>
 
 <!-- markdownlint-restore -->
 <!-- prettier-ignore-end -->

docs/_snippets/quickstart-package-install.mdx

@@ -46,7 +46,7 @@ Make sure to copy the packages that are relevant to your dbt version.
 ```yml packages.yml
 packages:
   - package: elementary-data/elementary
-    version: 0.7.4
+    version: 0.7.5
     ## Docs: https://docs.elementary-data.com
 ```
 
@@ -55,7 +55,7 @@ packages:
 ```yml packages.yml
 packages:
   - package: elementary-data/elementary
-    version: 0.7.4
+    version: 0.7.5
     ## Docs: https://docs.elementary-data.com
 
     ## !! Important !! For dbt >=1.2.0 \<1.3.0 ##
@@ -69,7 +69,7 @@ packages:
 ```yml packages.yml
 packages:
   - package: elementary-data/elementary
-    version: 0.7.4
+    version: 0.7.5
     ## Docs: https://docs.elementary-data.com
 
     ## !! Important !! For dbt <1.2.0 ##

elementary/config/config.py

@@ -78,6 +78,7 @@ def __init__(
             os.getcwd(),
         )
         os.makedirs(os.path.abspath(self.target_dir), exist_ok=True)
+        os.environ["DBT_LOG_PATH"] = os.path.abspath(target_path)
 
         self.update_bucket_website = self._first_not_none(
             update_bucket_website,

elementary/monitor/data_monitoring/data_monitoring.py

@@ -123,17 +123,31 @@ def get_latest_invocation(self) -> Dict[str, Any]:
             return {}
 
     @staticmethod
-    def _check_dbt_package_compatibility(dbt_pkg_ver: str):
+    def _check_dbt_package_compatibility(dbt_pkg_ver: str) -> None:
         dbt_pkg_ver = version.parse(dbt_pkg_ver)
         py_pkg_ver = version.parse(package.get_package_version())
-        logger.info(
-            f"Checking compatibility between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver})."
-        )
-        if (
-            dbt_pkg_ver.major != py_pkg_ver.major
-            or dbt_pkg_ver.minor != py_pkg_ver.minor
+        if dbt_pkg_ver.major > py_pkg_ver.major or (
+            dbt_pkg_ver.major == py_pkg_ver.major
+            and dbt_pkg_ver.minor > py_pkg_ver.minor
+        ):
+            logger.warning(
+                f"You are using incompatible versions between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}).\n "
+                "To fix please run:\n"
+                "pip install --upgrade elementary-data\n",
+            )
+            return
+
+        if dbt_pkg_ver.major < py_pkg_ver.major or (
+            dbt_pkg_ver.major == py_pkg_ver.major
+            and dbt_pkg_ver.minor < py_pkg_ver.minor
         ):
             logger.warning(
                 f"You are using incompatible versions between edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}).\n "
-                "Please upgrade the major and minor versions to align.\n",
+                "To fix please update your packages.yml, and run:\n"
+                "dbt deps && dbt run --select elementary\n",
             )
+            return
+
+        logger.info(
+            f"edr ({py_pkg_ver}) and Elementary's dbt package ({dbt_pkg_ver}) are compatible :)"
+        )

elementary/monitor/dbt_project/macros/get_elementary_table_nodes.sql

@@ -1,5 +1,5 @@
 {% macro get_elementary_table_nodes() %}
     {% set table_materializations = ["table", "incremental"] %}
-    {% set table_nodes = graph.nodes.values() | selectattr('config.materialized', 'in', table_materializations) %}
-    {% do elementary.edr_log(tojson(table_nodes | list)) %}
+    {% set table_nodes = graph.nodes.values() | selectattr('package_name', '==', 'elementary') | selectattr('config.materialized', 'in', table_materializations) %}
+    {% do return(table_nodes | list) %}
 {% endmacro %}

elementary/monitor/dbt_project/packages.yml

@@ -2,4 +2,4 @@ packages:
   - package: dbt-labs/dbt_utils
     version: [">=0.8.0", "<0.9.0"]
   - package: elementary-data/elementary
-    version: 0.7.4
+    version: 0.7.5

elementary/monitor/dbt_project_utils.py

@@ -16,7 +16,20 @@
 _PACKAGES_PATH = os.path.join(PATH, "dbt_packages")
 
 
-def get_elementary_package_path():
+def is_dbt_package_up_to_date() -> bool:
+    installed_version = _get_installed_dbt_package_version()
+    if installed_version is None:
+        return False
+
+    required_version = _get_required_dbt_package_version()
+    # We're using a non dbt Hub requirement, such as Git or local.
+    if not required_version:
+        return True
+
+    return installed_version == required_version
+
+
+def _get_elementary_package_path():
     package_path = os.path.join(_PACKAGES_PATH, _DBT_PACKAGE_NAME)
     if os.path.exists(package_path):
         return package_path
@@ -28,17 +41,8 @@ def get_elementary_package_path():
     return None
 
 
-def is_dbt_package_up_to_date() -> bool:
-    installed_version = get_installed_dbt_package_version()
-    if installed_version is None:
-        return False
-
-    required_version = get_required_dbt_package_version()
-    return installed_version == required_version
-
-
-def get_installed_dbt_package_version() -> Optional[str]:
-    package_path = get_elementary_package_path()
+def _get_installed_dbt_package_version() -> Optional[str]:
+    package_path = _get_elementary_package_path()
     if package_path is None:
         return None
 
@@ -50,13 +54,15 @@ def get_installed_dbt_package_version() -> Optional[str]:
     return project_yaml_dict["version"]
 
 
-def get_required_dbt_package_version() -> Optional[str]:
+def _get_required_dbt_package_version() -> Optional[str]:
     packages_file_path = os.path.join(PATH, _PACKAGES_FILENAME)
     packages_yaml = OrderedYaml().load(packages_file_path)
 
-    for requirement in packages_yaml["packages"]:
-        package_name = requirement["package"].split("/")[-1]
+    for requirement in packages_yaml.get("packages", []):
+        package_id = requirement.get("package")
+        if not package_id:
+            continue
+        package_name = package_id.split("/")[-1]
         if package_name == _DBT_PACKAGE_NAME:
             return requirement["version"]
-
     return None

elementary/monitor/fetchers/models/schema.py

@@ -25,13 +25,13 @@ def format_generated_at(cls, generated_at):
 
 
 class ArtifactSchema(ExtendedBaseModel):
-    name: str
-    unique_id: str
-    owners: List[str]
-    tags: List[str]
-    package_name: Optional[str]
-    description: Optional[str]
-    full_path: str
+    name: Optional[str] = None
+    unique_id: Optional[str] = None
+    owners: Optional[List[str]] = None
+    tags: Optional[List[str]] = None
+    package_name: Optional[str] = None
+    description: Optional[str] = None
+    full_path: Optional[str] = None
 
     @validator("tags", pre=True)
     def load_tags(cls, tags):
@@ -47,22 +47,25 @@ def format_full_path_sep(cls, full_path: str) -> str:
 
 
 class ModelSchema(ArtifactSchema):
-    database_name: str = None
+    database_name: Optional[str] = None
     schema_name: str
     table_name: str
+    ref_function: str = "ref"
 
 
 class SourceSchema(ArtifactSchema):
-    database_name: str = None
+    database_name: Optional[str] = None
     schema_name: str
     table_name: str
+    ref_function: str = "source"
 
 
 class ExposureSchema(ArtifactSchema):
-    url: Optional[str]
-    type: Optional[str]
-    maturity: Optional[str]
-    owner_email: Optional[str]
+    url: Optional[str] = None
+    type: Optional[str] = None
+    maturity: Optional[str] = None
+    owner_email: Optional[str] = None
+    depends_on: Optional[List[str]] = None
 
 
 class ModelTestCoverage(ExtendedBaseModel):

elementary/operations/cli.py

@@ -39,6 +39,12 @@ def run_operation():
     help="Which directory to look in for the profiles.yml file. "
     "If not set, edr will look in the current working directory first, then HOME/.dbt/",
 )
+@click.option(
+    "--target-path",
+    type=str,
+    default=Config.DEFAULT_TARGET_PATH,
+    help="Absolute target path for saving edr files such as logs and reports",
+)
 @click.pass_context
 def upload_source_freshness(ctx, **conf):
     """