From f2e8801da4aa99fcc627d1266d369bfc3944ae81 Mon Sep 17 00:00:00 2001 From: Michelle Brier Date: Fri, 9 Aug 2024 12:40:49 -0700 Subject: [PATCH 1/3] fix --- .../src/tasks/repair_audio_analyses.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/discovery-provider/src/tasks/repair_audio_analyses.py b/packages/discovery-provider/src/tasks/repair_audio_analyses.py index 6be959401bf..88cfe14b016 100644 --- a/packages/discovery-provider/src/tasks/repair_audio_analyses.py +++ b/packages/discovery-provider/src/tasks/repair_audio_analyses.py @@ -5,6 +5,7 @@ import requests from redis import Redis from sqlalchemy import or_ +from sqlalchemy.orm import load_only from sqlalchemy.orm.session import Session from src.models.tracks.track import Track @@ -32,6 +33,16 @@ def valid_bpm(bpm): def query_tracks(session: Session) -> List[Track]: tracks = ( session.query(Track) + .options( + # Eagerly load necessary attributes to avoid DetachedInstanceError + load_only( + Track.track_cid, + Track.audio_upload_id, + Track.musical_key, + Track.bpm, + Track.audio_analysis_error_count, + ) + ) .filter( Track.is_current == True, or_(Track.musical_key == None, Track.bpm == None), From ecd57bef614d5a44cdd61075365f43c5c066b278 Mon Sep 17 00:00:00 2001 From: Michelle Brier Date: Fri, 9 Aug 2024 13:47:03 -0700 Subject: [PATCH 2/3] fix again --- .../src/tasks/repair_audio_analyses.py | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/packages/discovery-provider/src/tasks/repair_audio_analyses.py b/packages/discovery-provider/src/tasks/repair_audio_analyses.py index 88cfe14b016..5d83f296196 100644 --- a/packages/discovery-provider/src/tasks/repair_audio_analyses.py +++ b/packages/discovery-provider/src/tasks/repair_audio_analyses.py @@ -4,7 +4,7 @@ import requests from redis import Redis -from sqlalchemy import or_ +from sqlalchemy import and_, or_ from sqlalchemy.orm import load_only from sqlalchemy.orm.session import Session @@ -36,16 +36,21 @@ def query_tracks(session: Session) -> List[Track]: .options( # Eagerly load necessary attributes to avoid DetachedInstanceError load_only( + Track.track_id, Track.track_cid, Track.audio_upload_id, Track.musical_key, Track.bpm, + Track.is_custom_bpm, Track.audio_analysis_error_count, ) ) .filter( Track.is_current == True, - or_(Track.musical_key == None, Track.bpm == None), + or_( + Track.musical_key == None, + and_(Track.bpm == None, Track.is_custom_bpm == False), + ), Track.audio_analysis_error_count < 3, Track.track_cid != None, Track.genre != "Podcasts", @@ -66,10 +71,12 @@ def select_content_nodes(redis: Redis): return random.sample(endpoints, min(5, len(endpoints))) -def repair(db, redis: Redis): +def repair(session: Session, redis: Redis): # Query batch of tracks that are missing key or bpm and have err counts < 3 from db - with db.scoped_session() as session: - tracks = query_tracks(session) + tracks = query_tracks(session) + session.commit() # Close tx + session.expunge_all() # Detach all instances so they can be referenced without a session + nodes = select_content_nodes(redis) num_tracks_updated = 0 for track in tracks: @@ -109,7 +116,7 @@ def repair(db, redis: Redis): if valid_musical_key(key): track_updated = True track.musical_key = key - if bpm and not track.bpm: + if bpm and not track.bpm and not track.is_custom_bpm: if valid_bpm(bpm): track_updated = True track.bpm = bpm @@ -117,11 +124,18 @@ def repair(db, redis: Redis): track_updated = True track.audio_analysis_error_count = error_count if track_updated: - num_tracks_updated += 1 - - with db.scoped_session() as session: - session.merge(track) - session.commit() + # Update track in a tx + try: + session.merge(track) + session.commit() + num_tracks_updated += 1 + except Exception as e: + session.rollback() + logger.error( + f"repair_audio_analyses.py | Error committing track update for track ID {track.track_id}", + exc_info=True, + ) + raise e if error_count >= 3: logger.warning( @@ -152,7 +166,8 @@ def repair_audio_analyses(self) -> None: have_lock = update_lock.acquire(blocking=False) if have_lock: try: - repair(db, redis) + with db.scoped_session(expire_on_commit=False) as session: + repair(session, redis) except Exception as e: logger.error( "repair_audio_analyses.py | Fatal error in main loop", exc_info=True From a4c3e282383bf65a1d2a57a33c9927b6d22810b5 Mon Sep 17 00:00:00 2001 From: Michelle Brier Date: Fri, 9 Aug 2024 13:51:51 -0700 Subject: [PATCH 3/3] remove manual rollback, scoped_session should take care of it --- packages/discovery-provider/src/tasks/repair_audio_analyses.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/discovery-provider/src/tasks/repair_audio_analyses.py b/packages/discovery-provider/src/tasks/repair_audio_analyses.py index 5d83f296196..2004967e7ea 100644 --- a/packages/discovery-provider/src/tasks/repair_audio_analyses.py +++ b/packages/discovery-provider/src/tasks/repair_audio_analyses.py @@ -130,7 +130,6 @@ def repair(session: Session, redis: Redis): session.commit() num_tracks_updated += 1 except Exception as e: - session.rollback() logger.error( f"repair_audio_analyses.py | Error committing track update for track ID {track.track_id}", exc_info=True,