Sp6 to Sp7 add missing constraints and tables

.github/workflows/test.yml

@@ -73,6 +73,11 @@ jobs:
         options:
           --health-cmd="mariadb-admin ping" --health-interval=5s
           --health-timeout=2s --health-retries=3
+      redis:
+        image: redis:latest
+        ports:
+          - 6379
+
 
     steps:
       - uses: actions/checkout@v4
@@ -128,6 +133,8 @@ jobs:
           echo "MIGRATOR_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
           echo "APP_USER_NAME = 'MasterUser'" >> specifyweb/settings/local_specify_settings.py
           echo "APP_USER_PASSWORD = 'MasterPassword'" >> specifyweb/settings/local_specify_settings.py
+          echo "REDIS_HOST = '127.0.0.1'" >> specifyweb/settings/local_specify_settings.py
+          echo "REDIS_PORT = ${{ job.services.redis.ports[6379] }}" >> specifyweb/settings/local_specify_settings.py
 
       - name: Need these files to be present
         run:

specifyweb/backend/redis_cache/store.py

@@ -1,4 +1,4 @@
-from .utils import _set_string, _get_string
+from .utils import _set_string, _get_string, _delete_key, _add_to_set, _set_elements, _redis_type
 
 
 def set_string(key: str, value: str, time_to_live=None, override_existing=True):
@@ -15,3 +15,18 @@ def get_string(key: str, delete_key=False) -> str:
 
 def get_bytes(key: str, delete_key=False) -> bytes:
     return _get_string(key, delete_key=delete_key, decode_responses=False)
+
+
+def add_to_set(key: str, *elements: str):
+    return _add_to_set(key, *elements)
+
+def set_members(key: str):
+    return _set_elements(key)
+
+
+def delete_key(key: str):
+    return _delete_key(key)
+
+
+def redis_type(key: str):
+    return _redis_type(key)

specifyweb/backend/redis_cache/utils.py

@@ -1,4 +1,4 @@
-from typing import overload
+from typing import overload, Literal
 
 from redis import Redis
 from django.conf import settings
@@ -12,6 +12,9 @@ def redis_connection(decode_responses=True):
         raise ValueError("Redis is not correctly configured", redis_host, redis_port)
     return Redis(host=redis_host, port=redis_port, db=redis_db_index, decode_responses=decode_responses)
 
+def _delete_key(key: str):
+    host = redis_connection()
+    host.delete(key)
 
 def _set_string(key: str, value: str, time_to_live=None, override_existing=True, decode_responses=True):
     host = redis_connection(decode_responses=decode_responses)
@@ -37,3 +40,20 @@ def _get_string(key: str, delete_key: bool=False, decode_responses=True) -> str
         return host.getdel(key)
 
     return host.get(key)
+
+def _add_to_set(key: str, *elements: str):
+    if len(elements) <= 0:
+        return 0
+    host = redis_connection(decode_responses=True)
+    return host.sadd(key, *elements)
+
+def _set_elements(key: str) -> set:
+    host = redis_connection(decode_responses=True)
+    return host.smembers(key)
+
+# https://redis.io/docs/latest/commands/type/
+Redis_Type = Literal["none", "string", "list", "set", "hash", "stream", "vectorset"]
+
+def _redis_type(key: str) -> Redis_Type:
+    host = redis_connection(decode_responses=True)
+    return host.type(key)

specifyweb/frontend/js_src/lib/components/Interactions/InteractionDialog.tsx

@@ -80,7 +80,8 @@ export function InteractionDialog({
   );
 
   const isLoanReturnLike =
-    isLoanReturn || (actionTable.name !== 'Loan' && actionTable.name.includes('Loan'));
+    isLoanReturn ||
+    (actionTable.name !== 'Loan' && actionTable.name.includes('Loan'));
 
   const itemTable = isLoanReturnLike ? tables.Loan : tables.CollectionObject;
 
@@ -206,8 +207,7 @@ export function InteractionDialog({
             )
         ).then((data) =>
           availablePrepsReady(catalogNumbers, data, {
-            skipEntryMatch:
-              searchField.name.toLowerCase() !== 'catalognumber',
+            skipEntryMatch: searchField.name.toLowerCase() !== 'catalognumber',
           })
         )
       );
@@ -377,7 +377,9 @@ export function InteractionDialog({
             values,
             isLoan
           )
-      ).then((data) => availablePrepsReady(values, data, { skipEntryMatch: true }))
+      ).then((data) =>
+        availablePrepsReady(values, data, { skipEntryMatch: true })
+      )
     );
   }
 

specifyweb/specify/datamodel.py

@@ -1,5 +1,5 @@
 from .models_utils.load_datamodel import Table, Field, Relationship, IdField, Datamodel, Index, add_collectingevents_to_locality, \
-    flag_dependent_fields, flag_system_tables, DoesNotExistError, TableDoesNotExistError, FieldDoesNotExistError
+    flag_dependent_fields, flag_system_tables, DoesNotExistError, TableDoesNotExistError, FieldDoesNotExistError, ManyToMany
 
 # These datamodel classes were generated by the specifyweb.specify.sp7_build_datamodel.py script.
 # The original models in this file are based on the Specify 6.8.03 datamodel schema.
@@ -815,10 +815,10 @@ def is_tree_table(table: Table):
             Index(name='SchemeNameIDX', column_names=['SchemeName'])
         ],
         relationships=[
-            Relationship(name='collections', type='many-to-many',required=False, relatedModelName='Collection', otherSideName='numberingSchemes'),
+            ManyToMany(name='collections', required=False, relatedModelName="Collection", otherSideName="numberingSchemes", through_model="Autonumschcoll", through_field="autonumberingscheme"),
             Relationship(name='createdByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='CreatedByAgentID'),
-            Relationship(name='disciplines', type='many-to-many',required=False, relatedModelName='Discipline', otherSideName='numberingSchemes'),
-            Relationship(name='divisions', type='many-to-many',required=False, relatedModelName='Division', otherSideName='numberingSchemes'),
+            ManyToMany(name='disciplines', required=False, relatedModelName="Discipline", otherSideName="numberingSchemes", through_model="Autonumschdsp", through_field="autonumberingscheme"),
+            ManyToMany(name='divisions', required=False, relatedModelName="Division", otherSideName="numberingSchemes", through_model="Autonumschdiv", through_field="autonumberingscheme"),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID')
         ],
         fieldAliases=[
@@ -1482,7 +1482,7 @@ def is_tree_table(table: Table):
             Relationship(name='institutionNetwork', type='many-to-one',required=False, relatedModelName='Institution', column='InstitutionNetworkID'),
             Relationship(name='leftSideRelTypes', type='one-to-many',required=False, relatedModelName='CollectionRelType', otherSideName='leftSideCollection'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
-            Relationship(name='numberingSchemes', type='many-to-many',required=False, relatedModelName='AutoNumberingScheme', otherSideName='collections'),
+            ManyToMany(name='numberingSchemes', required=False, relatedModelName="AutoNumberingScheme", otherSideName="collections", through_model="Autonumschcoll", through_field="collection"),
             Relationship(name='pickLists', type='one-to-many',required=False, relatedModelName='PickList', otherSideName='collection'),
             Relationship(name='prepTypes', type='one-to-many',required=False, relatedModelName='PrepType', otherSideName='collection'),
             Relationship(name='rightSideRelTypes', type='one-to-many',required=False, relatedModelName='CollectionRelType', otherSideName='rightSideCollection'),
@@ -1601,7 +1601,7 @@ def is_tree_table(table: Table):
             Relationship(name='otherIdentifiers', type='one-to-many',required=False, relatedModelName='OtherIdentifier', otherSideName='collectionObject', dependent=True),
             Relationship(name='paleoContext', type='many-to-one',required=False, relatedModelName='PaleoContext', column='PaleoContextID', otherSideName='collectionObjects'),
             Relationship(name='preparations', type='one-to-many',required=False, relatedModelName='Preparation', otherSideName='collectionObject', dependent=True),
-            Relationship(name='projects', type='many-to-many',required=False, relatedModelName='Project', otherSideName='collectionObjects'),
+            ManyToMany(name='projects', required=False, relatedModelName="Project", otherSideName="collectionObjects", through_model="Project_colobj", through_field="collectionobject"),
             Relationship(name='rightSideRels', type='one-to-many',required=False, relatedModelName='CollectionRelationship', otherSideName='rightSide', dependent=True),
             Relationship(name='treatmentEvents', type='one-to-many',required=False, relatedModelName='TreatmentEvent', otherSideName='collectionObject', dependent=True),
             Relationship(name='visibilitySetBy', type='many-to-one',required=False, relatedModelName='SpecifyUser', column='VisibilitySetByID'),
@@ -3064,7 +3064,7 @@ def is_tree_table(table: Table):
             Relationship(name='lithoStratTreeDef', type='many-to-one',required=False, relatedModelName='LithoStratTreeDef', column='LithoStratTreeDefID', otherSideName='disciplines'),
             Relationship(name='tectonicUnitTreeDef', type='many-to-one',required=False, relatedModelName='TectonicUnitTreeDef', column='TectonicUnitTreeDefID', otherSideName='disciplines'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
-            Relationship(name='numberingSchemes', type='many-to-many',required=False, relatedModelName='AutoNumberingScheme', otherSideName='disciplines'),
+            ManyToMany(name='numberingSchemes', required=False, relatedModelName="AutoNumberingScheme", otherSideName="disciplines", through_model="Autonumschdsp", through_field="discipline"),
             Relationship(name='spExportSchemas', type='one-to-many',required=False, relatedModelName='SpExportSchema', otherSideName='discipline'),
             Relationship(name='spLocaleContainers', type='one-to-many',required=False, relatedModelName='SpLocaleContainer', otherSideName='discipline'),
             Relationship(name='userGroups', type='one-to-many',required=False, relatedModelName='SpPrincipal', otherSideName='scope')
@@ -3236,7 +3236,7 @@ def is_tree_table(table: Table):
             Relationship(name='institution', type='many-to-one',required=True, relatedModelName='Institution', column='InstitutionID', otherSideName='divisions'),
             Relationship(name='members', type='one-to-many',required=False, relatedModelName='Agent', otherSideName='division'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
-            Relationship(name='numberingSchemes', type='many-to-many',required=False, relatedModelName='AutoNumberingScheme', otherSideName='divisions'),
+            ManyToMany(name='numberingSchemes', required=False, relatedModelName="AutoNumberingScheme", otherSideName="divisions", through_model="Autonumschdiv", through_field="division"),
             Relationship(name='userGroups', type='one-to-many',required=False, relatedModelName='SpPrincipal', otherSideName='scope')
         ],
         fieldAliases=[
@@ -5900,7 +5900,7 @@ def is_tree_table(table: Table):
         ],
         relationships=[
             Relationship(name='agent', type='many-to-one',required=False, relatedModelName='Agent', column='ProjectAgentID'),
-            Relationship(name='collectionObjects', type='many-to-many',required=False, relatedModelName='CollectionObject', otherSideName='projects'),
+            ManyToMany(name='collectionObjects', required=False, relatedModelName="CollectionObject", otherSideName="projects", through_model="Project_colobj", through_field="project"),
             Relationship(name='createdByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='CreatedByAgentID'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID')
         ],
@@ -6360,7 +6360,7 @@ def is_tree_table(table: Table):
             Relationship(name='discipline', type='many-to-one',required=True, relatedModelName='Discipline', column='DisciplineID', otherSideName='spExportSchemas'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
             Relationship(name='spExportSchemaItems', type='one-to-many',required=False, relatedModelName='SpExportSchemaItem', otherSideName='spExportSchema'),
-            Relationship(name='spExportSchemaMappings', type='many-to-many',required=False, relatedModelName='SpExportSchemaMapping', otherSideName='spExportSchemas')
+            ManyToMany(name='spExportSchemaMappings', required=False, relatedModelName="SpExportSchemaMapping", otherSideName="spExportSchemas", through_model="Spexportschema_exportmapping", through_field="spexportschema"),
         ],
         fieldAliases=[
 
@@ -6451,7 +6451,7 @@ def is_tree_table(table: Table):
             Relationship(name='createdByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='CreatedByAgentID'),
             Relationship(name='mappings', type='one-to-many',required=False, relatedModelName='SpExportSchemaItemMapping', otherSideName='exportSchemaMapping'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
-            Relationship(name='spExportSchemas', type='many-to-many',required=False, relatedModelName='SpExportSchema', otherSideName='spExportSchemaMappings'),
+            ManyToMany(name='spExportSchemas', required=False, relatedModelName="SpExportSchema", otherSideName="spExportSchemaMappings", through_model="Spexportschema_exportmapping", through_field="spexportschemamapping"),
             Relationship(name='symbiotaInstances', type='one-to-many',required=False, relatedModelName='SpSymbiotaInstance', otherSideName='schemaMapping')
         ],
         fieldAliases=[
@@ -6613,7 +6613,7 @@ def is_tree_table(table: Table):
 
         ],
         relationships=[
-            Relationship(name='principals', type='many-to-many',required=False, relatedModelName='SpPrincipal', otherSideName='permissions')
+            ManyToMany(name='principals', required=False, relatedModelName="SpPrincipal", otherSideName="permissions", through_model="Spprincipal_sppermission", through_field="sppermission"),
         ],
         fieldAliases=[
 
@@ -6643,9 +6643,9 @@ def is_tree_table(table: Table):
         relationships=[
             Relationship(name='createdByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='CreatedByAgentID'),
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
-            Relationship(name='permissions', type='many-to-many',required=False, relatedModelName='SpPermission', otherSideName='principals'),
+            ManyToMany(name='permissions', required=False, relatedModelName="SpPermission", otherSideName="principals", through_model="Spprincipal_sppermission", through_field="spprincipal"),
             Relationship(name='scope', type='many-to-one',required=False, relatedModelName='UserGroupScope', column='userGroupScopeID', otherSideName='userGroups'),
-            Relationship(name='specifyUsers', type='many-to-many',required=False, relatedModelName='SpecifyUser', otherSideName='spPrincipals')
+            ManyToMany(name='specifyUsers', required=False, relatedModelName="SpecifyUser", otherSideName="spPrincipals", through_model="Specifyuser_spprincipal", through_field="spprincipal"),
         ],
         fieldAliases=[
 
@@ -6954,7 +6954,7 @@ def is_tree_table(table: Table):
             Relationship(name='modifiedByAgent', type='many-to-one',required=False, relatedModelName='Agent', column='ModifiedByAgentID'),
             Relationship(name='spAppResourceDirs', type='one-to-many',required=False, relatedModelName='SpAppResourceDir', otherSideName='specifyUser'),
             Relationship(name='spAppResources', type='one-to-many',required=False, relatedModelName='SpAppResource', otherSideName='specifyUser'),
-            Relationship(name='spPrincipals', type='many-to-many',required=False, relatedModelName='SpPrincipal', otherSideName='specifyUsers'),
+            ManyToMany(name='spPrincipals', required=False, relatedModelName="SpPrincipal", otherSideName="specifyUsers", through_model="Specifyuser_spprincipal", through_field="specifyuser"),
             Relationship(name='spQuerys', type='one-to-many',required=False, relatedModelName='SpQuery', otherSideName='specifyUser'),
             Relationship(name='taskSemaphores', type='one-to-many',required=False, relatedModelName='SpTaskSemaphore', otherSideName='owner'),
             Relationship(name='workbenchTemplates', type='one-to-many',required=False, relatedModelName='WorkbenchTemplate', otherSideName='specifyUser'),