From 40a46d5f8c37f9618bf097b6fc4202b9a584e88b Mon Sep 17 00:00:00 2001 From: Leo Moll Date: Fri, 5 Jan 2018 19:44:18 +0100 Subject: [PATCH 1/3] Improved Dominik's dangerous optimizations by implementing database recovery functions --- classes/exceptions.py | 9 + classes/storesqlite.py | 420 ++++++++++++++++++++++------------------- classes/updater.py | 10 + 3 files changed, 243 insertions(+), 196 deletions(-) create mode 100644 classes/exceptions.py diff --git a/classes/exceptions.py b/classes/exceptions.py new file mode 100644 index 0000000..9aed17e --- /dev/null +++ b/classes/exceptions.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Leo Moll +# + +class DatabaseCurrupted( RuntimeError ): + """This exception is raised when the database throws errors during update""" + +class DatabaseLost( RuntimeError ): + """This exception is raised when the connection to the database is lost during update""" diff --git a/classes/storesqlite.py b/classes/storesqlite.py index cbad1dc..2ce005d 100644 --- a/classes/storesqlite.py +++ b/classes/storesqlite.py @@ -5,6 +5,8 @@ # -- Imports ------------------------------------------------ import os, stat, string, sqlite3, time +from classes.exceptions import DatabaseCurrupted + # -- Classes ------------------------------------------------ class StoreSQLite( object ): def __init__( self, logger, notifier, settings ): @@ -27,12 +29,16 @@ def Init( self, reset = False ): self.logger.info( '===== RESET: Database will be deleted and regenerated =====' ) self._file_remove( self.dbfile ) self.conn = sqlite3.connect( self.dbfile ) - self.Reset() + self._handle_database_initialization() else: - self.conn = sqlite3.connect( self.dbfile ) - - self.conn.execute('pragma journal_mode=off') # 3x speed-up, check mode 'WAL' - self.conn.execute('pragma synchronous=off') # that is a bit dangerous :-) but faaaast + try: + self.conn = sqlite3.connect( self.dbfile ) + except sqlite3.DatabaseError as err: + self.logger.error( 'Errore while opening database. Trying to fully reset the Database...' ) + self.Init( reset = True ) + + self.conn.execute( 'pragma journal_mode=off' ) # 3x speed-up, check mode 'WAL' + self.conn.execute( 'pragma synchronous=off' ) # that is a bit dangerous :-) but faaaast self.conn.create_function( 'UNIX_TIMESTAMP', 0, UNIX_TIMESTAMP ) self.conn.create_aggregate( 'GROUP_CONCAT', 1, GROUP_CONCAT ) @@ -183,7 +189,6 @@ def GetStatus( self ): cursor.execute( 'SELECT * FROM `status` LIMIT 1' ) r = cursor.fetchall() cursor.close() - self.conn.commit() if len( r ) == 0: status['status'] = "NONE" return status @@ -333,132 +338,212 @@ def SupportsUpdate( self ): return True def ftInit( self ): - # prevent concurrent updating - cursor = self.conn.cursor() - cursor.execute( - """ - UPDATE `status` - SET `modified` = ?, - `status` = 'UPDATING' - WHERE ( `status` != 'UPDATING' ) - OR - ( `modified` < ? ) - """, ( - int( time.time() ), - int( time.time() ) - 86400 + try: + # prevent concurrent updating + self.conn.commit() + cursor = self.conn.cursor() + cursor.execute( + """ + UPDATE `status` + SET `modified` = ?, + `status` = 'UPDATING' + WHERE ( `status` != 'UPDATING' ) + OR + ( `modified` < ? ) + """, ( + int( time.time() ), + int( time.time() ) - 86400 + ) ) - ) - retval = cursor.rowcount > 0 - self.conn.commit() - cursor.close() - self.ft_channel = None - self.ft_channelid = None - self.ft_show = None - self.ft_showid = None - return retval + retval = cursor.rowcount > 0 + self.conn.commit() + cursor.close() + self.ft_channel = None + self.ft_channelid = None + self.ft_show = None + self.ft_showid = None + return retval + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def ftUpdateStart( self, full ): - cursor = self.conn.cursor() - if full: - cursor.executescript( """ - UPDATE `channel` - SET `touched` = 0; - - UPDATE `show` - SET `touched` = 0; - - UPDATE `film` - SET `touched` = 0; - """ ) - cursor.execute( 'SELECT COUNT(*) FROM `channel`' ) - r1 = cursor.fetchone() - cursor.execute( 'SELECT COUNT(*) FROM `show`' ) - r2 = cursor.fetchone() - cursor.execute( 'SELECT COUNT(*) FROM `film`' ) - r3 = cursor.fetchone() - cursor.close() - self.conn.commit() - return ( r1[0], r2[0], r3[0], ) + try: + cursor = self.conn.cursor() + if full: + cursor.executescript( """ + UPDATE `channel` + SET `touched` = 0; + + UPDATE `show` + SET `touched` = 0; + + UPDATE `film` + SET `touched` = 0; + """ ) + cursor.execute( 'SELECT COUNT(*) FROM `channel`' ) + r1 = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `show`' ) + r2 = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `film`' ) + r3 = cursor.fetchone() + cursor.close() + self.conn.commit() + return ( r1[0], r2[0], r3[0], ) + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def ftUpdateEnd( self, delete ): - cursor = self.conn.cursor() - cursor.execute( 'SELECT COUNT(*) FROM `channel` WHERE ( touched = 0 )' ) - ( del_chn, ) = cursor.fetchone() - cursor.execute( 'SELECT COUNT(*) FROM `show` WHERE ( touched = 0 )' ) - ( del_shw, ) = cursor.fetchone() - cursor.execute( 'SELECT COUNT(*) FROM `film` WHERE ( touched = 0 )' ) - ( del_mov, ) = cursor.fetchone() - if delete: - cursor.execute( 'DELETE FROM `show` WHERE ( show.touched = 0 ) AND ( ( SELECT SUM( film.touched ) FROM `film` WHERE film.showid = show.id ) = 0 )' ) - cursor.execute( 'DELETE FROM `film` WHERE ( touched = 0 )' ) - else: - del_chn = 0 - del_shw = 0 - del_mov = 0 - cursor.execute( 'SELECT COUNT(*) FROM `channel`' ) - ( cnt_chn, ) = cursor.fetchone() - cursor.execute( 'SELECT COUNT(*) FROM `show`' ) - ( cnt_shw, ) = cursor.fetchone() - cursor.execute( 'SELECT COUNT(*) FROM `film`' ) - ( cnt_mov, ) = cursor.fetchone() - cursor.close() - self.conn.commit() - return ( del_chn, del_shw, del_mov, cnt_chn, cnt_shw, cnt_mov, ) + try: + cursor = self.conn.cursor() + cursor.execute( 'SELECT COUNT(*) FROM `channel` WHERE ( touched = 0 )' ) + ( del_chn, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `show` WHERE ( touched = 0 )' ) + ( del_shw, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `film` WHERE ( touched = 0 )' ) + ( del_mov, ) = cursor.fetchone() + if delete: + cursor.execute( 'DELETE FROM `show` WHERE ( show.touched = 0 ) AND ( ( SELECT SUM( film.touched ) FROM `film` WHERE film.showid = show.id ) = 0 )' ) + cursor.execute( 'DELETE FROM `film` WHERE ( touched = 0 )' ) + else: + del_chn = 0 + del_shw = 0 + del_mov = 0 + cursor.execute( 'SELECT COUNT(*) FROM `channel`' ) + ( cnt_chn, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `show`' ) + ( cnt_shw, ) = cursor.fetchone() + cursor.execute( 'SELECT COUNT(*) FROM `film`' ) + ( cnt_mov, ) = cursor.fetchone() + cursor.close() + self.conn.commit() + return ( del_chn, del_shw, del_mov, cnt_chn, cnt_shw, cnt_mov, ) + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def ftInsertFilm( self, film ): - cursor = self.conn.cursor() - newchn = False - inschn = 0 - insshw = 0 - insmov = 0 - - # handle channel - if self.ft_channel != film['channel']: - # process changed channel - newchn = True - cursor.execute( 'SELECT `id`,`touched` FROM `channel` WHERE channel.channel=?', ( film['channel'], ) ) - r = cursor.fetchall() - if len( r ) > 0: - # get the channel data - self.ft_channel = film['channel'] - self.ft_channelid = r[0][0] - if r[0][1] == 0: - # updated touched - cursor.execute( 'UPDATE `channel` SET `touched`=1 WHERE ( channel.id=? )', ( self.ft_channelid, ) ) + try: + cursor = self.conn.cursor() + newchn = False + inschn = 0 + insshw = 0 + insmov = 0 + + # handle channel + if self.ft_channel != film['channel']: + # process changed channel + newchn = True + cursor.execute( 'SELECT `id`,`touched` FROM `channel` WHERE channel.channel=?', ( film['channel'], ) ) + r = cursor.fetchall() + if len( r ) > 0: + # get the channel data + self.ft_channel = film['channel'] + self.ft_channelid = r[0][0] + if r[0][1] == 0: + # updated touched + cursor.execute( 'UPDATE `channel` SET `touched`=1 WHERE ( channel.id=? )', ( self.ft_channelid, ) ) + self.conn.commit() + else: + # insert the new channel + inschn = 1 + cursor.execute( 'INSERT INTO `channel` ( `dtCreated`,`channel` ) VALUES ( ?,? )', ( int( time.time() ), film['channel'] ) ) + self.ft_channel = film['channel'] + self.ft_channelid = cursor.lastrowid self.conn.commit() - else: - # insert the new channel - inschn = 1 - cursor.execute( 'INSERT INTO `channel` ( `dtCreated`,`channel` ) VALUES ( ?,? )', ( int( time.time() ), film['channel'] ) ) - self.ft_channel = film['channel'] - self.ft_channelid = cursor.lastrowid - self.conn.commit() - # handle show - if newchn or self.ft_show != film['show']: - # process changed show - cursor.execute( 'SELECT `id`,`touched` FROM `show` WHERE ( show.channelid=? ) AND ( show.show=? )', ( self.ft_channelid, film['show'] ) ) + # handle show + if newchn or self.ft_show != film['show']: + # process changed show + cursor.execute( 'SELECT `id`,`touched` FROM `show` WHERE ( show.channelid=? ) AND ( show.show=? )', ( self.ft_channelid, film['show'] ) ) + r = cursor.fetchall() + if len( r ) > 0: + # get the show data + self.ft_show = film['show'] + self.ft_showid = r[0][0] + if r[0][1] == 0: + # updated touched + cursor.execute( 'UPDATE `show` SET `touched`=1 WHERE ( show.id=? )', ( self.ft_showid, ) ) + self.conn.commit() + else: + # insert the new show + insshw = 1 + cursor.execute( + """ + INSERT INTO `show` ( + `dtCreated`, + `channelid`, + `show`, + `search` + ) + VALUES ( + ?, + ?, + ?, + ? + ) + """, ( + int( time.time() ), + self.ft_channelid, film['show'], + self._make_search( film['show'] ) + ) + ) + self.ft_show = film['show'] + self.ft_showid = cursor.lastrowid + self.conn.commit() + + # check if the movie is there + cursor.execute( """ + SELECT `id`, + `touched` + FROM `film` + WHERE ( film.channelid = ? ) + AND + ( film.showid = ? ) + AND + ( film.url_video = ? ) + """, ( self.ft_channelid, self.ft_showid, film['url_video'] ) ) r = cursor.fetchall() if len( r ) > 0: - # get the show data - self.ft_show = film['show'] - self.ft_showid = r[0][0] + # film found + filmid = r[0][0] if r[0][1] == 0: - # updated touched - cursor.execute( 'UPDATE `show` SET `touched`=1 WHERE ( show.id=? )', ( self.ft_showid, ) ) + # update touched + cursor.execute( 'UPDATE `film` SET `touched`=1 WHERE ( film.id=? )', ( filmid, ) ) self.conn.commit() else: - # insert the new show - insshw = 1 + # insert the new film + insmov = 1 cursor.execute( """ - INSERT INTO `show` ( + INSERT INTO `film` ( `dtCreated`, `channelid`, - `show`, - `search` + `showid`, + `title`, + `search`, + `aired`, + `duration`, + `size`, + `description`, + `website`, + `url_sub`, + `url_video`, + `url_video_sd`, + `url_video_hd` ) VALUES ( + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, + ?, ?, ?, ?, @@ -466,93 +551,36 @@ def ftInsertFilm( self, film ): ) """, ( int( time.time() ), - self.ft_channelid, film['show'], - self._make_search( film['show'] ) + self.ft_channelid, + self.ft_showid, + film['title'], + self._make_search( film['title'] ), + film['airedepoch'], + self._make_duration( film['duration'] ), + film['size'], + film['description'], + film['website'], + film['url_sub'], + film['url_video'], + film['url_video_sd'], + film['url_video_hd'] ) ) - self.ft_show = film['show'] - self.ft_showid = cursor.lastrowid - self.conn.commit() - - # check if the movie is there - cursor.execute( """ - SELECT `id`, - `touched` - FROM `film` - WHERE ( film.channelid = ? ) - AND - ( film.showid = ? ) - AND - ( film.url_video = ? ) - """, ( self.ft_channelid, self.ft_showid, film['url_video'] ) ) - r = cursor.fetchall() - if len( r ) > 0: - # film found - filmid = r[0][0] - if r[0][1] == 0: - # update touched - cursor.execute( 'UPDATE `film` SET `touched`=1 WHERE ( film.id=? )', ( filmid, ) ) + filmid = cursor.lastrowid self.conn.commit() - else: - # insert the new film - insmov = 1 - cursor.execute( - """ - INSERT INTO `film` ( - `dtCreated`, - `channelid`, - `showid`, - `title`, - `search`, - `aired`, - `duration`, - `size`, - `description`, - `website`, - `url_sub`, - `url_video`, - `url_video_sd`, - `url_video_hd` - ) - VALUES ( - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ?, - ? - ) - """, ( - int( time.time() ), - self.ft_channelid, - self.ft_showid, - film['title'], - self._make_search( film['title'] ), - film['airedepoch'], - self._make_duration( film['duration'] ), - film['size'], - film['description'], - film['website'], - film['url_sub'], - film['url_video'], - film['url_video_sd'], - film['url_video_hd'] - ) - ) - filmid = cursor.lastrowid - self.conn.commit() - cursor.close() - return ( filmid, inschn, insshw, insmov ) - - def Reset( self ): + cursor.close() + return ( filmid, inschn, insshw, insmov ) + except sqlite3.DatabaseError as err: + self._handle_database_corruption( err ) + raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + + def _handle_database_corruption( self, err ): + self.logger.error( 'Database error during critical operation: {} - Database will be rebuilt from scratch.', err ) + self.notifier.ShowDatabaseError( err ) + self.Exit() + self.Init( reset = True ) + + def _handle_database_initialization( self ): self.conn.executescript( """ PRAGMA foreign_keys = false; diff --git a/classes/updater.py b/classes/updater.py index 7d2005b..086f3b1 100644 --- a/classes/updater.py +++ b/classes/updater.py @@ -8,6 +8,8 @@ from operator import itemgetter from classes.store import Store +from classes.exceptions import DatabaseCurrupted +from classes.exceptions import DatabaseLost # -- Constants ---------------------------------------------- FILMLISTE_AKT_URL = 'https://res.mediathekview.de/akt.xml' @@ -143,6 +145,14 @@ def Import( self, full ): self.logger.info( 'Import of {} finished', destfile ) self.notifier.CloseUpdateProgress() return True + except DatabaseCorrupted as err: + self.logger.error( '{}', err ) + self.notifier.CloseUpdateProgress() + file.close() + except DatabaseLost as err: + self.logger.error( '{}', err ) + self.notifier.CloseUpdateProgress() + file.close() except IOError as err: self.logger.error( 'Error {} wile processing {}', err, destfile ) try: From 616f20ae944dea653a59000dd844496e4a0a04e9 Mon Sep 17 00:00:00 2001 From: Leo Moll Date: Fri, 5 Jan 2018 19:54:29 +0100 Subject: [PATCH 2/3] Last update time not updated when update aborted --- addon.xml | 13 ++++++------- classes/exceptions.py | 2 +- classes/updater.py | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/addon.xml b/addon.xml index 921d75f..df397aa 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ @@ -27,12 +27,11 @@ Ermöglicht den Zugriff auf fast alle deutschen Mediatheken der öffentlich Rechtlichen basierend auf der Datenbank von MediathekView Gives access to most video-platforms from German public service broadcasters using the database of MediathekView Fornisce l'accesso a gran parte delle piattaforme video operate dalle emittenti pubbliche tedesche usando la banca dati di MediathekView - v0.2.7 (2018-01-05): -- Fixed Naming Exception during update -- Updated English README -- Added German README -- Added Italian README -- Added LICENSE (MIT) + v0.2.8 (2018-01-05): +- Last update time not updated when update aborted +- Dramatic speedup of update with SQLite +- Handling of database corruption + all de fr MIT License diff --git a/classes/exceptions.py b/classes/exceptions.py index 9aed17e..53d7f14 100644 --- a/classes/exceptions.py +++ b/classes/exceptions.py @@ -2,7 +2,7 @@ # Copyright 2017 Leo Moll # -class DatabaseCurrupted( RuntimeError ): +class DatabaseCorrupted( RuntimeError ): """This exception is raised when the database throws errors during update""" class DatabaseLost( RuntimeError ): diff --git a/classes/updater.py b/classes/updater.py index 086f3b1..c989aef 100644 --- a/classes/updater.py +++ b/classes/updater.py @@ -317,7 +317,7 @@ def _update_end( self, full, status ): self.logger.info( 'Total: channels:%d, shows:%d, movies:%d' % ( self.tot_chn, self.tot_shw, self.tot_mov ) ) self.db.UpdateStatus( status, - int( time.time() ), + int( time.time() ) if status != 'ABORTED' else None, None, 1 if full else 0, self.add_chn, self.add_shw, self.add_mov, From e14f6214ead5bf4501b4b6da2a042f77afb11b4a Mon Sep 17 00:00:00 2001 From: Leo Moll Date: Fri, 5 Jan 2018 19:58:43 +0100 Subject: [PATCH 3/3] Fixed class spelling --- classes/storesqlite.py | 10 +++++----- classes/updater.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/classes/storesqlite.py b/classes/storesqlite.py index 2ce005d..5fed712 100644 --- a/classes/storesqlite.py +++ b/classes/storesqlite.py @@ -5,7 +5,7 @@ # -- Imports ------------------------------------------------ import os, stat, string, sqlite3, time -from classes.exceptions import DatabaseCurrupted +from classes.exceptions import DatabaseCorrupted # -- Classes ------------------------------------------------ class StoreSQLite( object ): @@ -365,7 +365,7 @@ def ftInit( self ): return retval except sqlite3.DatabaseError as err: self._handle_database_corruption( err ) - raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def ftUpdateStart( self, full ): try: @@ -392,7 +392,7 @@ def ftUpdateStart( self, full ): return ( r1[0], r2[0], r3[0], ) except sqlite3.DatabaseError as err: self._handle_database_corruption( err ) - raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def ftUpdateEnd( self, delete ): try: @@ -421,7 +421,7 @@ def ftUpdateEnd( self, delete ): return ( del_chn, del_shw, del_mov, cnt_chn, cnt_shw, cnt_mov, ) except sqlite3.DatabaseError as err: self._handle_database_corruption( err ) - raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def ftInsertFilm( self, film ): try: @@ -572,7 +572,7 @@ def ftInsertFilm( self, film ): return ( filmid, inschn, insshw, insmov ) except sqlite3.DatabaseError as err: self._handle_database_corruption( err ) - raise DatabaseCurrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) + raise DatabaseCorrupted( 'Database error during critical operation: {} - Database will be rebuilt from scratch.'.format( err ) ) def _handle_database_corruption( self, err ): self.logger.error( 'Database error during critical operation: {} - Database will be rebuilt from scratch.', err ) diff --git a/classes/updater.py b/classes/updater.py index c989aef..e57fed5 100644 --- a/classes/updater.py +++ b/classes/updater.py @@ -8,7 +8,7 @@ from operator import itemgetter from classes.store import Store -from classes.exceptions import DatabaseCurrupted +from classes.exceptions import DatabaseCorrupted from classes.exceptions import DatabaseLost # -- Constants ----------------------------------------------