diff --git a/database/sqlite/sqlite_context.c b/database/sqlite/sqlite_context.c index 26ed8a96aab31f..5fad94be2f1121 100644 --- a/database/sqlite/sqlite_context.c +++ b/database/sqlite/sqlite_context.c @@ -52,7 +52,7 @@ int sql_init_context_database(int memory) if (likely(!memory)) target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION); - if (configure_sqlite_database(db_context_meta, target_version)) + if (configure_sqlite_database(db_context_meta, target_version, "context_config")) return 1; if (likely(!memory)) @@ -60,12 +60,12 @@ int sql_init_context_database(int memory) else snprintfz(buf, sizeof(buf) - 1, "ATTACH DATABASE ':memory:' as meta"); - if(init_database_batch(db_context_meta, list)) return 1; + if(init_database_batch(db_context_meta, list, "context")) return 1; - if (init_database_batch(db_context_meta, &database_context_config[0])) + if (init_database_batch(db_context_meta, &database_context_config[0], "context_init")) return 1; - if (init_database_batch(db_context_meta, &database_context_cleanup[0])) + if (init_database_batch(db_context_meta, &database_context_cleanup[0], "context_cleanup")) return 1; return 0; diff --git a/database/sqlite/sqlite_db_migration.c b/database/sqlite/sqlite_db_migration.c index 00a1e105584e5a..6ff047069d1cd0 100644 --- a/database/sqlite/sqlite_db_migration.c +++ b/database/sqlite/sqlite_db_migration.c @@ -156,14 +156,14 @@ const char *database_migrate_v13_v14[] = { static int do_migration_v1_v2(sqlite3 *database) { if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "hops")) - return init_database_batch(database, &database_migrate_v1_v2[0]); + return init_database_batch(database, &database_migrate_v1_v2[0], "meta_migrate"); return 0; } static int do_migration_v2_v3(sqlite3 *database) { if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "memory_mode")) - return init_database_batch(database, &database_migrate_v2_v3[0]); + return init_database_batch(database, &database_migrate_v2_v3[0], "meta_migrate"); return 0; } @@ -198,12 +198,12 @@ static int do_migration_v3_v4(sqlite3 *database) static int do_migration_v4_v5(sqlite3 *database) { - return init_database_batch(database, &database_migrate_v4_v5[0]); + return init_database_batch(database, &database_migrate_v4_v5[0], "meta_migrate"); } static int do_migration_v5_v6(sqlite3 *database) { - return init_database_batch(database, &database_migrate_v5_v6[0]); + return init_database_batch(database, &database_migrate_v5_v6[0], "meta_migrate"); } static int do_migration_v6_v7(sqlite3 *database) @@ -341,14 +341,14 @@ static int do_migration_v8_v9(sqlite3 *database) static int do_migration_v9_v10(sqlite3 *database) { if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "chart_labels")) - return init_database_batch(database, &database_migrate_v9_v10[0]); + return init_database_batch(database, &database_migrate_v9_v10[0], "meta_migrate"); return 0; } static int do_migration_v10_v11(sqlite3 *database) { if (table_exists_in_database(database, "health_log") && !column_exists_in_table(database, "health_log", "chart_name")) - return init_database_batch(database, &database_migrate_v10_v11[0]); + return init_database_batch(database, &database_migrate_v10_v11[0], "meta_migrate"); return 0; } @@ -360,7 +360,7 @@ static int do_migration_v11_v12(sqlite3 *database) if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary") && table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary")) - rc = init_database_batch(database, &database_migrate_v11_v12[0]); + rc = init_database_batch(database, &database_migrate_v11_v12[0], "meta_migrate"); if (!rc) sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL); @@ -400,12 +400,12 @@ static int do_migration_v12_v13(sqlite3 *database) int rc = 0; if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary")) { - rc = init_database_batch(database, &database_migrate_v12_v13_detail[0]); + rc = init_database_batch(database, &database_migrate_v12_v13_detail[0], "meta_migrate"); sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL); } if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary")) - rc = init_database_batch(database, &database_migrate_v12_v13_hash[0]); + rc = init_database_batch(database, &database_migrate_v12_v13_hash[0], "meta_migrate"); return rc; } @@ -413,7 +413,7 @@ static int do_migration_v12_v13(sqlite3 *database) static int do_migration_v13_v14(sqlite3 *database) { if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "last_connected")) - return init_database_batch(database, &database_migrate_v13_v14[0]); + return init_database_batch(database, &database_migrate_v13_v14[0], "meta_migrate"); return 0; } @@ -431,7 +431,7 @@ const char *database_ml_migrate_v1_v2[] = { static int do_ml_migration_v1_v2(sqlite3 *database) { if (get_auto_vaccum(database) != 2) - return init_database_batch(database, &database_ml_migrate_v1_v2[0]); + return init_database_batch(database, &database_ml_migrate_v1_v2[0], "ml_migrate"); return 0; } diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index f9f1217b59309c..a94b6f25c3539e 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -119,7 +119,7 @@ SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) { break; case SQLITE_BUSY: case SQLITE_LOCKED: - global_statistics_sqlite3_query_completed(rc == SQLITE_DONE, rc == SQLITE_BUSY, rc == SQLITE_LOCKED); + global_statistics_sqlite3_query_completed(false, rc == SQLITE_BUSY, rc == SQLITE_LOCKED); usleep(SQLITE_INSERT_DELAY * USEC_PER_MS); continue; default: @@ -201,7 +201,7 @@ int execute_insert(sqlite3_stmt *res) return rc; } -int configure_sqlite_database(sqlite3 *database, int target_version) +int configure_sqlite_database(sqlite3 *database, int target_version, const char *description) { char buf[1024 + 1] = ""; const char *list[2] = { buf, NULL }; @@ -209,42 +209,42 @@ int configure_sqlite_database(sqlite3 *database, int target_version) // https://www.sqlite.org/pragma.html#pragma_auto_vacuum // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL; snprintfz(buf, sizeof(buf) - 1, "PRAGMA auto_vacuum=%s", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL")); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; // https://www.sqlite.org/pragma.html#pragma_synchronous // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA; snprintfz(buf, sizeof(buf) - 1, "PRAGMA synchronous=%s", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL")); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; // https://www.sqlite.org/pragma.html#pragma_journal_mode // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_mode=%s", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL")); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; // https://www.sqlite.org/pragma.html#pragma_temp_store // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY; snprintfz(buf, sizeof(buf) - 1, "PRAGMA temp_store=%s", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY")); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; // https://www.sqlite.org/pragma.html#pragma_journal_size_limit // PRAGMA schema.journal_size_limit = N ; snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_size_limit=%lld", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216)); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; // https://www.sqlite.org/pragma.html#pragma_cache_size // PRAGMA schema.cache_size = pages; // PRAGMA schema.cache_size = -kibibytes; snprintfz(buf, sizeof(buf) - 1, "PRAGMA cache_size=%lld", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000)); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; snprintfz(buf, sizeof(buf) - 1, "PRAGMA user_version=%d", target_version); - if (init_database_batch(database, list)) + if (init_database_batch(database, list, description)) return 1; return 0; @@ -315,7 +315,21 @@ int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **state return rc; } -int init_database_batch(sqlite3 *database, const char *batch[]) +char *get_database_extented_error(sqlite3 *database, int i, const char *description) +{ + const char *err = sqlite3_errstr(sqlite3_extended_errcode(database)); + + if (!err) + return NULL; + + size_t len = strlen(err)+ strlen(description) + 32; + char *full_err = mallocz(len); + + snprintfz(full_err, len - 1, "%s: %d: %s", description, i, err); + return full_err; +} + +int init_database_batch(sqlite3 *database, const char *batch[], const char *description) { int rc; char *err_msg = NULL; @@ -324,8 +338,11 @@ int init_database_batch(sqlite3 *database, const char *batch[]) if (rc != SQLITE_OK) { error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg); error_report("SQLite failed statement %s", batch[i]); - analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(sqlite3_extended_errcode(database))); + char *error_str = get_database_extented_error(database, i, description); + if (error_str) + analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str); sqlite3_free(err_msg); + freez(error_str); if (SQLITE_CORRUPT == rc) { if (mark_database_to_recover(NULL, database)) error_report("Database is corrupted will attempt to fix"); @@ -408,7 +425,10 @@ int sql_init_database(db_check_action_type_t rebuild, int memory) rc = sqlite3_open(sqlite_database, &db_meta); if (rc != SQLITE_OK) { error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); - analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(sqlite3_extended_errcode(db_meta))); + char *error_str = get_database_extented_error(db_meta, 0, "meta_open"); + if (error_str) + analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str); + freez(error_str); sqlite3_close(db_meta); db_meta = NULL; return 1; @@ -447,13 +467,13 @@ int sql_init_database(db_check_action_type_t rebuild, int memory) if (likely(!memory)) target_version = perform_database_migration(db_meta, DB_METADATA_VERSION); - if (configure_sqlite_database(db_meta, target_version)) + if (configure_sqlite_database(db_meta, target_version, "meta_config")) return 1; - if (init_database_batch(db_meta, &database_config[0])) + if (init_database_batch(db_meta, &database_config[0], "meta_init")) return 1; - if (init_database_batch(db_meta, &database_cleanup[0])) + if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup")) return 1; netdata_log_info("SQLite database initialization completed"); diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index 9cd1f7ad473f27..b9ec89353ce046 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -47,10 +47,10 @@ SQLITE_API int sqlite3_exec_monitored( ); // Initialization and shutdown -int init_database_batch(sqlite3 *database, const char *batch[]); +int init_database_batch(sqlite3 *database, const char *batch[], const char *description); int sql_init_database(db_check_action_type_t rebuild, int memory); void sql_close_database(void); -int configure_sqlite_database(sqlite3 *database, int target_version); +int configure_sqlite_database(sqlite3 *database, int target_version, const char *description); // Helpers int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null); diff --git a/ml/ml.cc b/ml/ml.cc index c6d077e289f2c1..91a40a6b247c15 100644 --- a/ml/ml.cc +++ b/ml/ml.cc @@ -1787,7 +1787,7 @@ void ml_init() // create table if (db) { int target_version = perform_ml_database_migration(db, ML_METADATA_VERSION); - if (configure_sqlite_database(db, target_version)) { + if (configure_sqlite_database(db, target_version, "ml_config")) { error_report("Failed to setup ML database"); sqlite3_close(db); db = NULL;