Include the bibliography files (bibtex) in the Database

echemdb/data/cv/database.py

@@ -56,12 +56,27 @@ class Database:
         0
 
     """
-    def __init__(self, data_packages=None):
+    def __init__(self, data_packages=None, bibliography=None):
         if data_packages is None:
             import os.path
             import echemdb.data.remote
             data_packages = echemdb.data.remote.collect_datapackages(os.path.join('website-gh-pages', 'data', 'generated', 'svgdigitizer'))
+
+            if bibliography is None:
+                bibliography = echemdb.data.remote.collect_bibliography(os.path.join('website-gh-pages', 'Literature'))
+
+        if bibliography is None:
+            bibliography = []
+
+        from collections.abc import Iterable
+        if isinstance(bibliography, Iterable):
+            from pybtex.database import BibliographyData
+            bibliography = BibliographyData(entries={
+                entry.key: entry for entry in bibliography
+            })
+
         self._packages = data_packages
+        self._bibliography = bibliography
 
     @classmethod
     def create_example(self):
@@ -78,7 +93,31 @@ def create_example(self):
         entries = Entry.create_examples("alves_2011_electrochemistry_6010") + \
                         Entry.create_examples("engstfeld_2018_polycrystalline_17743")
 
-        return Database([entry.package for entry in entries])
+        return Database([entry.package for entry in entries], [entry.bibliography for entry in entries])
+
+    @property
+    def bibliography(self):
+        r"""
+        Return a pybtex database of all bibtex bibliography files.
+
+        EXAMPLES::
+
+            >>> database = Database.create_example()
+            >>> database.bibliography
+            BibliographyData(
+              entries=OrderedCaseInsensitiveDict([
+                ('alves_2011_electrochemistry_6010', Entry('article',
+                ...
+                ('engstfeld_2018_polycrystalline_17743', Entry('article',
+                ...
+
+        """
+        from pybtex.database import BibliographyData
+
+        return BibliographyData({
+            entry.bibliography.key: entry.bibliography for entry in self if entry.bibliography
+        })
+
 
     def filter(self, predicate):
         r"""
@@ -91,7 +130,7 @@ def filter(self, predicate):
             [Entry('alves_2011_electrochemistry_6010_p2_2a_solid')]
 
         """
-        return Database([entry.package for entry in self if predicate(entry)])
+        return Database(data_packages=[entry.package for entry in self if predicate(entry)], bibliography=self._bibliography)
 
     def __iter__(self):
         r"""
@@ -105,7 +144,12 @@ def __iter__(self):
 
         """
         from echemdb.data.cv.entry import Entry
-        return iter([Entry(package) for package in self._packages])
+
+        def get_bibliography(package):
+            bib = Entry(package, bibliography=None).source.bib
+            return self._bibliography.entries.get(bib, None)
+
+        return iter([Entry(package, bibliography=get_bibliography(package)) for package in self._packages])
 
     def __len__(self):
         r"""

echemdb/data/cv/entry.py

@@ -41,8 +41,9 @@ class Entry:
         >>> entry = next(iter(database))
 
     """
-    def __init__(self, package):
+    def __init__(self, package, bibliography):
         self.package = package
+        self.bibliography = bibliography
 
     @property
     def identifier(self):
@@ -68,7 +69,7 @@ def __dir__(self):
 
             >>> entry = Entry.create_examples()[0]
             >>> dir(entry)
-            ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_descriptor', 'create_examples', 'curator', 'df', 'electrochemical_system', 'figure_description', 'identifier', 'package', 'plot', 'profile', 'resources', 'source', 'yaml']
+            ['__class__', '__delattr__', '__dict__', '__dir__', '__doc__', '__eq__', '__format__', '__ge__', '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__', '__init__', '__init_subclass__', '__le__', '__lt__', '__module__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', '__repr__', '__setattr__', '__sizeof__', '__str__', '__subclasshook__', '__weakref__', '_descriptor', 'bibliography', 'create_examples', 'curator', 'df', 'electrochemical_system', 'figure_description', 'identifier', 'package', 'plot', 'profile', 'resources', 'source', 'yaml']
 
         """
         return list(set(dir(Descriptor(self.package.descriptor)) + object.__dir__(self)))
@@ -214,13 +215,16 @@ def create_examples(cls, name="alves_2011_electrochemistry_6010"):
                 from svgdigitizer.__main__ import cv
                 invoke(cv, "--sampling_interval", ".005", "--package", "--metadata", yaml, svg, "--outdir", outdir)
 
-        from echemdb.data.local import collect_datapackages
+        from echemdb.data.local import collect_datapackages, collect_bibliography
         packages = collect_datapackages(outdir)
+        bibliography = collect_bibliography(source)
+        assert len(bibliography) == 1, f"No bibliography found for {name}."
+        bibliography = next(iter(bibliography))
 
         if len(packages) == 0:
             raise ValueError(f"No literature data found for {name}. There is probably some outdated data in {outdir}.")
 
-        return [Entry(package) for package in packages]
+        return [Entry(package=package, bibliography=bibliography) for package in packages]
 
 
 class Descriptor:

echemdb/data/local.py

@@ -42,3 +42,19 @@ def collect_datapackages(data):
     # Read the package descriptors (does not read the actual data CSVs)
     from datapackage import Package
     return [Package(descriptor) for descriptor in descriptors]
+
+def collect_bibliography(bibfiles):
+    r"""
+    Return a list of bibliography data (pybtex) parsed from the bibtex files
+    in the directory `bibfiles` and its subdirectories.
+
+    EXAMPLES::
+
+        >>> bibfiles = collect_bibliography(".")
+
+    """
+    import os.path
+    from glob import glob
+    from pybtex.database import parse_file
+
+    return [entry for file in glob(os.path.join(bibfiles, '**', '*.bib'), recursive=True) for entry in parse_file(file, bib_format="bibtex").entries.values()]

echemdb/data/remote.py

@@ -25,6 +25,15 @@
 
 from functools import cache
 
+@cache
+def collect_zipfile_from_url(url):
+    from urllib.request import urlopen
+    response = urlopen(url)
+
+    from zipfile import ZipFile
+    from io import BytesIO
+    return ZipFile(BytesIO(response.read()))
+
 @cache
 def collect_datapackages(data=".", url="https://github.com/echemdb/website/archive/refs/heads/gh-pages.zip", outdir=None):
     r"""
@@ -46,15 +55,39 @@ def collect_datapackages(data=".", url="https://github.com/echemdb/website/archi
         outdir = tempfile.mkdtemp()
         atexit.register(lambda dirname: shutil.rmtree(dirname), outdir)
 
-    from urllib.request import urlopen
-    response = urlopen(url)
-
-    from zipfile import ZipFile
-    from io import BytesIO
-    compressed = ZipFile(BytesIO(response.read()))
+    compressed = collect_zipfile_from_url(url)
 
     compressed.extractall(outdir, members=[name for name in compressed.namelist() if name.endswith('.json') or name.endswith('.csv')])
 
     import os.path
     import echemdb.data.local
     return echemdb.data.local.collect_datapackages(os.path.join(outdir, data))
+
+@cache
+def collect_bibliography(data=".", url="https://github.com/echemdb/website/archive/refs/heads/gh-pages.zip", outdir=None):
+    r"""
+    Return a list of bibliography files (bibtex) in a remote location.
+
+    The default is to download the bibliography currently available on echemdb and
+    extract them to a temporary directory.
+
+    EXAMPLES::
+
+        >>> packages = collect_bibliography()
+
+    """
+    if outdir is None:
+        import tempfile
+        import atexit
+        import shutil
+
+        outdir = tempfile.mkdtemp()
+        atexit.register(lambda dirname: shutil.rmtree(dirname), outdir)
+
+    compressed = collect_zipfile_from_url(url)
+
+    compressed.extractall(outdir, members=[name for name in compressed.namelist() if name.endswith('.bib')])
+
+    import os.path
+    import echemdb.data.local
+    return echemdb.data.local.collect_bibliography(os.path.join(outdir, data))

echemdb/website/generator/database.py

@@ -33,4 +33,13 @@
         'data'))
 )
 
-cv = echemdb.data.cv.database.Database(packages)
+bibliography = echemdb.data.local.collect_bibliography(
+    os.path.normpath(os.path.join(
+        os.path.dirname(__file__),
+        '..',
+        '..',
+        '..',
+        'literature'))
+)
+
+cv = echemdb.data.cv.database.Database(packages, bibliography)

environment.yml

@@ -12,6 +12,7 @@ dependencies:
   - pip
   - plotly
   - pip
+  - pybtex
   - pytest
   - pyyaml
   - pip:

literature/briega-martos_2021_cation_XXX/briega-martos_2021_cation_XXX.bib

@@ -1,4 +1,4 @@
-@article{briega2021cation,
+@article{briega-martos_2021_cation_XXX,
   title={Cation Effects on Interfacial Water Structure and Hydrogen Peroxide Reduction on Pt (111)},
   author={Briega-Martos, Valent{\'\i}n and Sarabia, Francisco J and Climent, V{\'\i}ctor and Herrero, Enrique and Feliu, Juan M},
   journal={ACS Measurement Science Au},

literature/engstfeld_2018_polycrystalline_17743/engstfeld_2018_polycrystalline_17743.bib

@@ -1,4 +1,4 @@
-@article{engstfeld2018polycrystalline,
+@article{engstfeld_2018_polycrystalline_17743,
   title={Polycrystalline and single-crystal Cu electrodes: influence of experimental conditions on the electrochemical properties in alkaline media},
   author={Engstfeld, Albert K and Maagaard, Thomas and Horch, Sebastian and Chorkendorff, Ib and Stephens, Ifan EL},
   journal={Chem.-Eur. J},