Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions doc/news/slugify_names.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
**Removed:**

* Removed `unitpackage.local.collect_datapackage` since it is identical to frictionless `package = Package()`.
3 changes: 1 addition & 2 deletions unitpackage/collection.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,9 +343,8 @@ def from_local_file(cls, filename):
[Entry('engstfeld_2018_polycrystalline_17743_f4b_1')]

"""
from unitpackage.local import collect_datapackage

package = collect_datapackage(filename)
package = Package(filename)

return cls(package=package)

Expand Down
4 changes: 2 additions & 2 deletions unitpackage/entry.py
Original file line number Diff line number Diff line change
Expand Up @@ -766,9 +766,9 @@ def from_local(cls, filename):
Entry('no_bibliography')

"""
from unitpackage.local import collect_datapackage
from unitpackage.local import Package

package = collect_datapackage(filename)
package = Package(filename)

if len(package.resources) == 0:
raise ValueError(f"No resource available in '{filename}'")
Expand Down
50 changes: 15 additions & 35 deletions unitpackage/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import os
import os.path
from glob import glob
from pathlib import Path

import pandas as pd
from frictionless import Package, Resource, Schema
Expand Down Expand Up @@ -69,23 +68,6 @@ def create_df_resource(resource):
return df_resource


def collect_datapackage(filename):
r"""
Return a Data Package from a :param filename which must be a valid frictionless Data Package (JSON).

EXAMPLES::

>>> package = collect_datapackage("./examples/local/no_bibliography/no_bibliography.json")
>>> package # doctest: +NORMALIZE_WHITESPACE
{'resources': [{'name':
...

"""
package = Package(filename)

return package


def collect_resources(datapackages):
r"""
Return a list of resources from a list of Data Packages.
Expand Down Expand Up @@ -121,7 +103,7 @@ def collect_datapackages(data):
"""
packages = sorted(glob(os.path.join(data, "**", "*.json"), recursive=True))

return [collect_datapackage(package) for package in packages]
return [Package(package) for package in packages]


def create_unitpackage(csvname, metadata=None, fields=None):
Expand Down Expand Up @@ -166,24 +148,19 @@ def create_unitpackage(csvname, metadata=None, fields=None):

csv_basename = os.path.basename(csvname)

package = Package(
resources=[
Resource(
path=csv_basename,
basepath=os.path.dirname(csvname) or ".",
)
],
resource = Resource(
path=csv_basename,
basepath=os.path.dirname(csvname) or ".",
)
package.infer()

resource = package.get_resource(Path(csv_basename).stem.lower())
resource.infer()

resource.custom.setdefault("metadata", {})
resource.custom["metadata"].setdefault("echemdb", metadata)

if fields:
# Update fields in the Data Package describing the data in the CSV
package_schema = resource.schema
# Update fields in the Resource describing the data in the CSV
resource_schema = resource.schema
if not isinstance(fields, list):
raise ValueError(
"'fields' must be a list such as \
Expand All @@ -205,14 +182,15 @@ def create_unitpackage(csvname, metadata=None, fields=None):

new_fields = []
unspecified_fields = []
for name in package_schema.field_names:

for name in resource_schema.field_names:
if name in provided_schema.field_names:
new_fields.append(
provided_schema.get_field(name).to_dict()
| package_schema.get_field(name).to_dict()
| resource_schema.get_field(name).to_dict()
)
else:
new_fields.append(package_schema.get_field(name).to_dict())
new_fields.append(resource_schema.get_field(name).to_dict())

if len(unspecified_fields) != 0:
logger.warning(
Expand All @@ -221,15 +199,17 @@ def create_unitpackage(csvname, metadata=None, fields=None):

unused_provided_fields = []
for name in provided_schema.field_names:
if name not in package_schema.field_names:
if name not in resource_schema.field_names:
unused_provided_fields.append(name)
if len(unused_provided_fields) != 0:
logger.warning(
f"Fields with names {unused_provided_fields} was provided but does not appear in the field names of tabular resource {package_schema.field_names}."
f"Fields with names {unused_provided_fields} was provided but does not appear in the field names of tabular resource {resource_schema.field_names}."
)

resource.schema = Schema.from_descriptor({"fields": new_fields})

package = Package(resources=[resource])

return package


Expand Down