diff --git a/sr_port/anticipatory_freeze.h b/sr_port/anticipatory_freeze.h index 6b297b0c7..f929a0a9a 100644 --- a/sr_port/anticipatory_freeze.h +++ b/sr_port/anticipatory_freeze.h @@ -3,7 +3,7 @@ * Copyright (c) 2012-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2018-2020 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -335,6 +335,10 @@ MBSTART { \ #define DB_LSEEKWRITE_HANG(CSA) FALSE #endif +#ifdef DEBUG +GBLREF gtm_int8 ydb_skip_bml_num; /* Use "gtm_int8" since it is possible "block_id" type is not yet defined */ +#endif + /* #GTM_THREAD_SAFE : The below macro (DB_LSEEKWRITE) is thread-safe */ #define DB_LSEEKWRITE(CSA, UDI, DB_FN, FD, OFFSET, BUFF, SIZE, STATUS) \ MBSTART { \ @@ -344,6 +348,11 @@ MBSTART { \ assert(!CSA_LOCAL || !CSA_LOCAL->region || FILE_INFO(CSA_LOCAL->region)->grabbed_access_sem \ || !(CSA_LOCAL)->nl || !FROZEN_CHILLED(CSA_LOCAL) || FREEZE_LATCH_HELD(CSA_LOCAL)); \ DBG_CHECK_DIO_ALIGNMENT(UDI, OFFSET, BUFF, SIZE); \ + assert((0 == ydb_skip_bml_num) || (NULL == UDI) || (0 == OFFSET) \ + || ((BLK_ZERO_OFF(((unix_db_info *)UDI)->s_addrs.hdr->start_vbn) \ + + ((gtm_int8)BLKS_PER_LMAP * ((unix_db_info *)UDI)->s_addrs.hdr->blk_size)) > OFFSET) \ + || ((BLK_ZERO_OFF(((unix_db_info *)UDI)->s_addrs.hdr->start_vbn) \ + + (ydb_skip_bml_num * ((unix_db_info *)UDI)->s_addrs.hdr->blk_size)) <= OFFSET)); \ /* We should never write to a READ_ONLY db file header unless we hold standalone access on the db */ \ assert((0 != OFFSET) || !((sgmnt_data_ptr_t)BUFF)->read_only || (NULL == UDI) || UDI->grabbed_access_sem); \ DO_LSEEKWRITE(CSA_LOCAL, DB_FN, FD, OFFSET, BUFF, SIZE, STATUS, fake_db_enospc, DB_LSEEKWRITE_HANG(CSA), \ diff --git a/sr_port/bm_getfree.c b/sr_port/bm_getfree.c index 84300d41c..4268f48c5 100644 --- a/sr_port/bm_getfree.c +++ b/sr_port/bm_getfree.c @@ -70,6 +70,10 @@ GBLREF uint4 dollar_tlevel; GBLREF uint4 update_array_size, cumul_update_array_size; GBLREF unsigned int t_tries; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_DBBADFREEBLKCTR); error_def(ERR_DBMBMINCFREFIXED); @@ -89,6 +93,10 @@ block_id bm_getfree(block_id hint, boolean_t *blk_used, unsigned int cw_work, cw total_blks = (dba_mm == cs_data->acc_meth) ? cs_addrs->total_blks : cs_addrs->ti->total_blks; if (hint >= total_blks) /* for TP, hint can be > total_blks */ hint = 1; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= hint) && (hint < ydb_skip_bml_num)) + hint = ydb_skip_bml_num; +# endif hint_cycled = DIVIDE_ROUND_UP(total_blks, BLKS_PER_LMAP); hint_limit = DIVIDE_ROUND_DOWN(hint, BLKS_PER_LMAP); local_maps = hint_cycled + 2; /* for (up to) 2 wraps */ @@ -102,6 +110,10 @@ block_id bm_getfree(block_id hint, boolean_t *blk_used, unsigned int cw_work, cw { hint_cycled = hint_limit; hint = 1; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= hint) && (hint < ydb_skip_bml_num)) + hint = ydb_skip_bml_num; +# endif continue; } if (SS_NORMAL != (status = GDSFILEXT(cs_data->extension_size, total_blks, TRANS_IN_PROG_TRUE))) @@ -125,6 +137,13 @@ block_id bm_getfree(block_id hint, boolean_t *blk_used, unsigned int cw_work, cw continue; } bml *= BLKS_PER_LMAP; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= bml) && (bml < ydb_skip_bml_num)) + { + hint = ydb_skip_bml_num + 1; + continue; + } +# endif if (ROUND_DOWN2(hint, BLKS_PER_LMAP) != bml) { /* not within requested map */ if ((bml < hint) && (hint_cycled)) /* wrap? - second one should force an extend for sure */ @@ -271,6 +290,17 @@ boolean_t is_free_blks_ctr_ok(void) local_maps = DIVIDE_ROUND_UP(total_blks, BLKS_PER_LMAP); for (free_blocks = 0, free_bml = 0; free_bml < local_maps; free_bml++) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) + && (BLKS_PER_LMAP <= (BLKS_PER_LMAP * free_bml)) + && ((BLKS_PER_LMAP * free_bml) < ydb_skip_bml_num)) + { + free_bml = (ydb_skip_bml_num / BLKS_PER_LMAP) - 1; + /* - 1 to compensate the "free_bml++" done in "for" loop line */ + free_blocks += (ydb_skip_bml_num - BLKS_PER_LMAP) / BLKS_PER_LMAP * (BLKS_PER_LMAP - 1); + continue; + } +# endif bml = bmm_find_free((uint4)free_bml, (sm_uc_ptr_t)MM_ADDR(cs_data), local_maps); if (bml < free_bml) break; @@ -299,6 +329,7 @@ boolean_t is_free_blks_ctr_ok(void) assert(cs_addrs->ti->free_blocks == free_blocks); if (cs_addrs->ti->free_blocks != free_blocks) { + assert(FALSE); send_msg_csa(CSA_ARG(cs_addrs) VARLSTCNT(6) ERR_DBBADFREEBLKCTR, 4, DB_LEN_STR(gv_cur_region), &(cs_addrs->ti->free_blocks), &free_blocks); cs_addrs->ti->free_blocks = free_blocks; diff --git a/sr_port/dse_maps.c b/sr_port/dse_maps.c index 91bb55806..5234afa21 100644 --- a/sr_port/dse_maps.c +++ b/sr_port/dse_maps.c @@ -3,6 +3,9 @@ * Copyright (c) 2001-2021 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2023 YottaDB LLC and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -56,6 +59,10 @@ GBLREF short crash_count; GBLREF srch_hist dummy_hist; GBLREF uint4 update_array_size; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_DBRDONLY); error_def(ERR_DSEBLKRDFAIL); error_def(ERR_DSEFAIL); @@ -120,6 +127,13 @@ void dse_maps(void) dse_m_rest(blk, bml_list, bml_size, &csa->ti->free_blocks, TRUE); for (blk_index = 0, bml_index = 0; blk_index < total_blks; blk_index += bplmap, bml_index++) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP == blk_index)) + { + bml_index += (ydb_skip_bml_num - blk_index) / BLKS_PER_LMAP; + blk_index = ydb_skip_bml_num; + } +# endif t_begin_crit(ERR_DSEFAIL); CHECK_TN(csa, csd, csd->trans_hist.curr_tn); /* can issue rts_error TNTOOLARGE */ CWS_RESET; diff --git a/sr_port/dse_range.c b/sr_port/dse_range.c index 32d88f5c2..75abfd771 100644 --- a/sr_port/dse_range.c +++ b/sr_port/dse_range.c @@ -41,6 +41,10 @@ GBLREF short int patch_path_count; GBLREF sgmnt_addrs *cs_addrs; GBLREF VSIG_ATOMIC_T util_interrupt; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_CTRLC); error_def(ERR_DSEBLKRDFAIL); error_def(ERR_DSEINVALBLKID); @@ -103,6 +107,13 @@ void dse_range(void) DSE_GRAB_CRIT_AS_APPROPRIATE(was_crit, was_hold_onto_crit, nocrit_present, cs_addrs, gv_cur_region); for (blk = from; blk <= to ;blk++) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= blk) && (blk < ydb_skip_bml_num)) + { + blk = ydb_skip_bml_num - 1; + continue; + } +# endif if (util_interrupt) { DSE_REL_CRIT_AS_APPROPRIATE(was_crit, was_hold_onto_crit, nocrit_present, cs_addrs, gv_cur_region); diff --git a/sr_port/gbldefs.c b/sr_port/gbldefs.c index 8d158c6ad..3559912ae 100644 --- a/sr_port/gbldefs.c +++ b/sr_port/gbldefs.c @@ -1419,3 +1419,6 @@ GBLDEF boolean_t ydb_treat_sigusr2_like_sigusr1; /* set based on env var "ydb_tr GBLDEF int jobinterrupt_sig_num; /* Set to signal number that caused $ZINTERRUPT (SIGUSR1/SIGUSR2). * Used to derive the value of the ISV $ZYINTRSIG. */ +#ifdef DEBUG +GBLDEF block_id ydb_skip_bml_num; /* See comment in "sr_port/gtm_env_init.c" for purpose */ +#endif diff --git a/sr_port/gtm_env_init.c b/sr_port/gtm_env_init.c index bbb4d8ab4..fc61e67c3 100644 --- a/sr_port/gtm_env_init.c +++ b/sr_port/gtm_env_init.c @@ -3,7 +3,7 @@ * Copyright (c) 2004-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2017-2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2017-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -87,6 +87,10 @@ GBLREF int ydb_repl_filter_timeout;/* # of seconds that source server waits bef GBLREF boolean_t dollar_test_default; /* Default value taken by dollar_truth via dollar_test_default */ GBLREF boolean_t gtm_nofflf; /* Used to control "write #" behavior ref GTM-9136 */ +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + void gtm_env_init(void) { boolean_t ret, is_defined; @@ -223,6 +227,18 @@ void gtm_env_init(void) ret = ydb_logical_truth_value(YDBENVINDX_DIRTREE_COLLHDR_ALWAYS, FALSE, &is_defined); if (is_defined) TREF(ydb_dirtree_collhdr_always) = ret; /* if logical is not defined, the TREF takes the default value */ + /* ydb_test_4g_db_blks env var. If set to a non-zero value of N, it implies blocks from the 1st local bitmap + * to the N-1th local bitmap block are not allocated in the db file. After the 0th local bitmap block, the + * Nth local bitmap block is where block allocation continues. This helps test records in the database blocks + * that contain 8-byte block numbers where the higher order 4-byte is non-zero. This effectively tests all + * code (mostly added in GT.M V7.0-000 and merged into YottaDB r2.00) that handles 8-byte block numbers. + * A 0 value implies this scheme is disabled. + */ + ydb_skip_bml_num = (block_id)ydb_trans_numeric(YDBENVINDX_TEST_4G_DB_BLKS, &is_defined, IGNORE_ERRORS_TRUE, NULL); + if (!is_defined) + ydb_skip_bml_num = 0; + else + ydb_skip_bml_num *= BLKS_PER_LMAP; # endif /* GDS Block certification */ ret = ydb_logical_truth_value(YDBENVINDX_GDSCERT, FALSE, &is_defined); diff --git a/sr_port/mu_int_blk.c b/sr_port/mu_int_blk.c index b321b291d..65287b54c 100644 --- a/sr_port/mu_int_blk.c +++ b/sr_port/mu_int_blk.c @@ -147,7 +147,7 @@ GBLREF gtm_uint64_t mu_int_cum[CUM_TYPE_MAX][MAX_BT_DEPTH + 1]; GBLREF uint4 mu_int_offset[]; GBLREF uint4 mu_int_errknt; GBLREF block_id mu_int_path[]; -GBLREF int4 mu_int_blks_to_upgrd; +GBLREF block_id mu_int_blks_to_upgrd; GBLREF global_list *trees; GBLREF global_list *trees_tail; GBLREF gv_key *mu_end_key; diff --git a/sr_port/mu_int_init.c b/sr_port/mu_int_init.c index 3d7d041aa..188bfc241 100644 --- a/sr_port/mu_int_init.c +++ b/sr_port/mu_int_init.c @@ -3,7 +3,7 @@ * Copyright (c) 2001-2018 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2019-2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2019-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -91,6 +91,11 @@ boolean_t mu_int_init(void) fc->op = FC_READ; /* Do aligned reads if opened with O_DIRECT */ udi = FC2UDI(fc); +# ifdef DEBUG + udi->s_addrs.hdr = NULL; /* Clear previous value of udi->s_addrs.hdr as it would have been freed up + * inside STANDALONE macro invoked above. + */ +# endif tsd = udi->fd_opened_with_o_direct ? (sgmnt_data_ptr_t)(TREF(dio_buff)).aligned : &mu_int_data; fc->op_buff = (uchar_ptr_t)tsd; fc->op_len = SGMNT_HDR_LEN; @@ -99,6 +104,11 @@ boolean_t mu_int_init(void) /* Ensure "mu_int_data" is populated even if we did not directly read into it for the O_DIRECT case */ if (udi->fd_opened_with_o_direct) memcpy(&mu_int_data, (TREF(dio_buff)).aligned, SGMNT_HDR_LEN); +# ifdef DEBUG + udi->s_addrs.hdr = &mu_int_data; /* Now that we have read in the file header, point udi to it for the next dbfilop. + * This is used by the DB_LSEEKREAD macro if "ydb_skip_bml_num" is non-zero. + */ +# endif if (MASTER_MAP_SIZE_MAX < MASTER_MAP_SIZE(&mu_int_data) || native_size < DIVIDE_ROUND_UP(SGMNT_HDR_LEN + MASTER_MAP_SIZE(&mu_int_data), DISK_BLOCK_SIZE) + MIN_DB_BLOCKS) { diff --git a/sr_port/mu_int_maps.c b/sr_port/mu_int_maps.c index 15dc1412a..adf524263 100644 --- a/sr_port/mu_int_maps.c +++ b/sr_port/mu_int_maps.c @@ -3,7 +3,7 @@ * Copyright (c) 2001-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2022-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -43,7 +43,7 @@ GBLREF sgmnt_data mu_int_data; GBLREF block_id mu_int_path[]; GBLREF boolean_t tn_reset_this_reg; GBLREF int mu_int_plen; -GBLREF int4 mu_int_blks_to_upgrd; +GBLREF block_id mu_int_blks_to_upgrd; GBLREF int disp_map_errors; GBLREF int mu_map_errs; GBLREF int disp_trans_errors; @@ -54,6 +54,10 @@ GBLREF sgmnt_addrs *cs_addrs; GBLREF trans_num largest_tn; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_DBREADBM); error_def(ERR_DBLVLINC); error_def(ERR_DBMBSIZMN); @@ -119,6 +123,10 @@ void mu_int_maps(void) { assert(mapsize == mu_int_data.bplmap); blkno = mcnt * mu_int_data.bplmap; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (0 < blkno) && (blkno < ydb_skip_bml_num)) + continue; +# endif bml_busy(0, mu_int_locals + ((blkno * BML_BITS_PER_BLK) / BITS_PER_UCHAR)); blk_base = mu_int_read(blkno, &ondsk_blkver, &free_blk_base); /* ondsk_blkver set to GDSV4 or GDSV6 (GDSVCURR) */ if (!blk_base) diff --git a/sr_port/mu_reorg_upgrd_dwngrd.c b/sr_port/mu_reorg_upgrd_dwngrd.c index 36bae5663..ee8f57e23 100644 --- a/sr_port/mu_reorg_upgrd_dwngrd.c +++ b/sr_port/mu_reorg_upgrd_dwngrd.c @@ -3,7 +3,7 @@ * Copyright (c) 2005-2021 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2018-2020 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -89,6 +89,10 @@ GBLREF unsigned char cw_set_depth; GBLREF unsigned char cw_map_depth; GBLREF uint4 update_trans; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_BUFFLUFAILED); error_def(ERR_DBBTUWRNG); error_def(ERR_DBFILERR); @@ -328,6 +332,13 @@ void mu_reorg_upgrd_dwngrd(void) memset(&reorg_stats, 0, SIZEOF(reorg_stats)); /* initialize statistics for this region */ for (curbmp = start_bmp; curbmp <= last_bmp; curbmp += BLKS_PER_LMAP) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= curbmp) && (curbmp < ydb_skip_bml_num)) + { + curbmp = ydb_skip_bml_num - BLKS_PER_LMAP; + continue; + } +# endif if (mu_ctrly_occurred || mu_ctrlc_occurred) { status1 = ERR_MUNOFINISH; diff --git a/sr_port/mu_swap_blk.c b/sr_port/mu_swap_blk.c index fd9c718f6..692802545 100644 --- a/sr_port/mu_swap_blk.c +++ b/sr_port/mu_swap_blk.c @@ -3,6 +3,9 @@ * Copyright (c) 2001-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2023 YottaDB LLC and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -79,6 +82,10 @@ GBLREF unsigned int t_tries; GBLREF gv_key *gv_currkey; GBLREF hash_table_int8 cw_stagnate; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + /****************************************************************************************** Input Parameters: level: level of working block @@ -147,6 +154,13 @@ enum cdb_sc mu_swap_blk(int level, block_id *pdest_blk_id, kill_set *kill_set_pt { blk_was_free = FALSE; INCR_BLK_NUM(dest_blk_id); +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP < dest_blk_id) && (dest_blk_id < ydb_skip_bml_num)) + { + dest_blk_id = ydb_skip_bml_num; + continue; + } +# endif /* A Pre-order traversal should not cause a child block to go to its parent. * However, in case it happens because already the organization was like that or for any other reason, skip swap. * If we decide to swap, code below should be changed to take care of the special case. diff --git a/sr_port/mucregini.c b/sr_port/mucregini.c index edcfe3e32..08008dbc3 100644 --- a/sr_port/mucregini.c +++ b/sr_port/mucregini.c @@ -3,7 +3,7 @@ * Copyright (c) 2001-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2019 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2019-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -68,6 +68,10 @@ GBLREF sgmnt_data_ptr_t cs_data; GBLREF sgmnt_addrs *cs_addrs; GBLREF void (*mupip_exit_fp)(int4 errnum); +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_COLLATIONUNDEF); error_def(ERR_COLLTYPVERSION); error_def(ERR_DBFILERR); @@ -266,6 +270,11 @@ void mucregini(block_id blk_init_size) NUM_CRIT_ENTRY(csd) = seg->mutex_slots; csd->wcs_phase2_commit_wait_spincnt = WCS_PHASE2_COMMIT_DEFAULT_SPINCNT; csd->defer_allocate = seg->defer_allocate; +# ifdef DEBUG + /* Force-enable DEFER_ALLOCATE in case ydb_skip_bml_num is non-zero to avoid huge test runtimes */ + if (0 != ydb_skip_bml_num) + csd->defer_allocate = TRUE; +# endif csd->read_only = seg->read_only; time(&ctime); assert(SIZEOF(ctime) >= SIZEOF(int4)); @@ -283,6 +292,10 @@ void mucregini(block_id blk_init_size) bmm_init(); for (i = 0; i < blk_init_size ; i += csd->bplmap) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (0 < i) && (i < ydb_skip_bml_num)) + i = ydb_skip_bml_num; +# endif status = bml_init(i); if (status != SS_NORMAL) { diff --git a/sr_port/mupip_integ.c b/sr_port/mupip_integ.c index b64b1f721..803d26520 100644 --- a/sr_port/mupip_integ.c +++ b/sr_port/mupip_integ.c @@ -117,7 +117,7 @@ GBLDEF global_list *trees; GBLDEF sgmnt_data mu_int_data; GBLDEF unsigned char *mu_int_master; GBLDEF trans_num largest_tn; -GBLDEF int4 mu_int_blks_to_upgrd; +GBLDEF block_id mu_int_blks_to_upgrd; GBLDEF span_node_integ *sndata; GBLDEF boolean_t null_coll_type_err = FALSE; GBLDEF boolean_t null_coll_type; @@ -442,6 +442,13 @@ void mupip_integ(void) * verifications */ csd = &mu_int_data; +# ifdef DEBUG + fc = gv_cur_region->dyn.addr->file_cntl; + udi = FC2UDI(fc); + udi->s_addrs.hdr = csd; /* Needed by asserts in "DB_LSEEKREAD" macro in "dbfilop()" calls when + * "ydb_skip_bml_num" is non-zero. + */ +# endif } trees_tail = trees = (global_list *)malloc(SIZEOF(global_list)); memset(trees, 0, SIZEOF(global_list)); diff --git a/sr_port/muprec.h b/sr_port/muprec.h index 65e90034a..8924d1197 100644 --- a/sr_port/muprec.h +++ b/sr_port/muprec.h @@ -3,7 +3,7 @@ * Copyright (c) 2001-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2018-2021 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -1282,7 +1282,7 @@ uint4 mur_back_phase1(reg_ctl_list *rctl); uint4 mur_back_phase2(reg_ctl_list *rctl); uint4 mur_back_processing_one_region(mur_back_opt_t *mur_back_options); int4 mur_block_count_correct(reg_ctl_list *rctl); -int4 mur_blocks_free(reg_ctl_list *rctl); +block_id mur_blocks_free(reg_ctl_list *rctl); boolean_t mur_close_files(void); int4 mur_cre_file_extfmt(jnl_ctl_list *jctl, int recstat); void mur_close_file_extfmt(boolean_t in_mur_close_files); diff --git a/sr_port/mur_blocks_free.c b/sr_port/mur_blocks_free.c index c86446003..bb4dbc896 100644 --- a/sr_port/mur_blocks_free.c +++ b/sr_port/mur_blocks_free.c @@ -3,6 +3,9 @@ * Copyright (c) 2001-2021 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2023 YottaDB LLC and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -38,16 +41,20 @@ GBLREF gd_region *gv_cur_region; GBLREF sgmnt_data_ptr_t cs_data; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + #define BPL SIZEOF(int4)*8/BML_BITS_PER_BLK /* blocks masked by a int4 */ error_def(ERR_DBRDERR); error_def(ERR_DYNUPGRDFAIL); -int4 mur_blocks_free(reg_ctl_list *rctl) +block_id mur_blocks_free(reg_ctl_list *rctl) { int4 x; - block_id bnum, maps; - int mapsize, i, j, k, fcnt, status; + block_id bnum, maps, i, fcnt; + int mapsize, j, k, status; unsigned char *c, *disk, *m_ptr; uint4 *dskmap, map_blk_size; file_control *db_ctl; @@ -75,6 +82,14 @@ int4 mur_blocks_free(reg_ctl_list *rctl) db_ctl->op_len = cs_data->blk_size; for (i = 0; i != maps; i++) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (1 == i)) + { + i = ydb_skip_bml_num / BLKS_PER_LMAP - 1; + fcnt += (ydb_skip_bml_num - BLKS_PER_LMAP) / BLKS_PER_LMAP * (BLKS_PER_LMAP - 1); + continue; + } +# endif bnum = i * cs_data->bplmap; db_ctl->op = FC_READ; db_ctl->op_pos = cs_data->start_vbn + ((gtm_int64_t)cs_data->blk_size / DISK_BLOCK_SIZE * bnum); diff --git a/sr_port/mur_process_intrpt_recov.c b/sr_port/mur_process_intrpt_recov.c index 43f82083b..0eaef7065 100644 --- a/sr_port/mur_process_intrpt_recov.c +++ b/sr_port/mur_process_intrpt_recov.c @@ -3,7 +3,7 @@ * Copyright (c) 2003-2017 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2018 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -63,7 +63,8 @@ uint4 mur_process_intrpt_recov() char prev_jnl_fn[MAX_FN_LEN + 1], rename_fn[MAX_FN_LEN + 1]; jnl_create_info jnl_info; uint4 status, status2; - uint4 max_autoswitchlimit, max_jnl_alq, max_jnl_deq, freeblks; + uint4 max_autoswitchlimit, max_jnl_alq, max_jnl_deq; + block_id freeblks; sgmnt_addrs *csa; sgmnt_data_ptr_t csd; jnl_private_control *jpc; diff --git a/sr_port/ydb_logicals_tab.h b/sr_port/ydb_logicals_tab.h index 3060c70c0..e3452f08b 100644 --- a/sr_port/ydb_logicals_tab.h +++ b/sr_port/ydb_logicals_tab.h @@ -1,6 +1,6 @@ /**************************************************************** * * - * Copyright (c) 2018-2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -178,6 +178,7 @@ YDBENVINDX_TABLE_ENTRY (YDBENVINDX_DB_COUNTER_SEM_INCR, "$ydb_db_counter_ YDBENVINDX_TABLE_ENTRY (YDBENVINDX_DIRTREE_COLLHDR_ALWAYS, "$ydb_dirtree_collhdr_always", "$gtm_dirtree_collhdr_always") YDBENVINDX_TABLE_ENTRY (YDBENVINDX_GVUNDEF_FATAL, "$ydb_gvundef_fatal", "$gtm_gvundef_fatal") YDBENVINDX_TABLE_ENTRY (YDBENVINDX_LOCKHASH_N_BITS, "$ydb_lockhash_n_bits", "") +YDBENVINDX_TABLE_ENTRY (YDBENVINDX_TEST_4G_DB_BLKS, "$ydb_test_4g_db_blks", "") YDBENVINDX_TABLE_ENTRY (YDBENVINDX_TEST_AUTORELINK_ALWAYS, "$ydb_test_autorelink_always", "$gtm_test_autorelink_always") YDBENVINDX_TABLE_ENTRY (YDBENVINDX_TEST_FAKE_ENOSPC, "$ydb_test_fake_enospc", "$gtm_test_fake_enospc") YDBENVINDX_TABLE_ENTRY (YDBENVINDX_TEST_JNLPOOL_SYNC, "$ydb_test_jnlpool_sync", "$gtm_test_jnlpool_sync") diff --git a/sr_unix/db_write_eof_block.c b/sr_unix/db_write_eof_block.c index 1e5c56e23..000317d72 100644 --- a/sr_unix/db_write_eof_block.c +++ b/sr_unix/db_write_eof_block.c @@ -3,6 +3,9 @@ * Copyright (c) 2016 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2023 YottaDB LLC and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -25,6 +28,10 @@ #include "gtmio.h" #include "anticipatory_freeze.h" +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + /* #GTM_THREAD_SAFE : The below function db_write_eof_block is thread-safe */ int db_write_eof_block(unix_db_info *udi, int fd, int blk_size, off_t offset, dio_buff_t *diobuff) { @@ -48,7 +55,21 @@ int db_write_eof_block(unix_db_info *udi, int fd, int blk_size, off_t offset, di { csa = &udi->s_addrs; assert(fd == udi->fd); +# ifdef DEBUG + block_id save_ydb_skip_bml_num; + + /* It is possible we write an EOF block to the offset corresponding to bitmap block BLKS_PER_LMAP. In that case, + * we want to disable the "ydb_skip_bml_num" related assert in DB_LSEEKWRITE. We do that by resetting + * "ydb_skip_bml_num" to 0 in that case. + */ + save_ydb_skip_bml_num = ydb_skip_bml_num; + if (offset == (BLK_ZERO_OFF(udi->s_addrs.hdr->start_vbn) + (BLKS_PER_LMAP * udi->s_addrs.hdr->blk_size))) + ydb_skip_bml_num = 0; +# endif DB_LSEEKWRITE(csa, udi, udi->fn, fd, offset, buff, blk_size, status); +# ifdef DEBUG + ydb_skip_bml_num = save_ydb_skip_bml_num; /* restore "ydb_skip_bml_num" in case it got reset */ +# endif } else DB_LSEEKWRITE(NULL, ((unix_db_info *)NULL), NULL, fd, offset, buff, blk_size, status); if (free_needed) diff --git a/sr_unix/dsk_read.c b/sr_unix/dsk_read.c index 5f78dc326..b620a1b52 100644 --- a/sr_unix/dsk_read.c +++ b/sr_unix/dsk_read.c @@ -65,6 +65,10 @@ GBLREF jnl_gbls_t jgbl; GBLREF uint4 process_id; GBLREF uint4 mu_reorg_encrypt_in_prog; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_DYNUPGRDFAIL); int4 dsk_read (block_id blk, sm_uc_ptr_t buff, enum db_ver *ondsk_blkver, boolean_t blk_free) @@ -121,6 +125,18 @@ int4 dsk_read (block_id blk, sm_uc_ptr_t buff, enum db_ver *ondsk_blkver, boolea assert(csd == cs_data); size = csd->blk_size; assert(csd->acc_meth == dba_bg); +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= blk) && (blk < ydb_skip_bml_num)) + { /* Huge db scheme is in effect and a block in the range of the HOLE in the huge db (any block + * number from the bitmap block BLKS_PER_LMAP to block "ydb_skip_bml_num-1") was specified. This is possible + * in case of a restartable transaction situation. In this case, just return a zeroed block and the + * transaction will eventually restart. + */ + memset(buff, 0, size); + in_dsk_read--; + return 0; + } +# endif /* Since csd->fully_upgraded is referenced more than once in this module (once explicitly and once in * GDS_BLK_UPGRADE_IF_NEEDED macro used below), take a copy of it and use that so all usages see the same value. * Not doing this, for example, can cause us to see the database as fully upgraded in the first check causing us diff --git a/sr_unix/gdsfilext.c b/sr_unix/gdsfilext.c index 942b0496b..4950c9574 100644 --- a/sr_unix/gdsfilext.c +++ b/sr_unix/gdsfilext.c @@ -81,7 +81,15 @@ MBSTART { \ 2, LEN_AND_LIT("Please make more disk space available or shutdown GT.M to avoid data loss"), ENOSPC); \ } MBEND -#define SUSPICIOUS_EXTEND (2 * (dollar_tlevel ? sgm_info_ptr->cw_set_depth : cw_set_depth) < cs_addrs->ti->free_blocks) +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + +#define SUSPICIOUS_EXTEND ((2 * (dollar_tlevel ? sgm_info_ptr->cw_set_depth : cw_set_depth) \ + DEBUG_ONLY( + ((0 != ydb_skip_bml_num) \ + ? ((ydb_skip_bml_num - BLKS_PER_LMAP) / BLKS_PER_LMAP * (BLKS_PER_LMAP - 1)) \ + : 0))) \ + < cs_addrs->ti->free_blocks) \ GBLREF sigset_t blockalrm; GBLREF sgmnt_addrs *cs_addrs; @@ -100,6 +108,10 @@ GBLREF boolean_t ydb_dbfilext_syslog_disable; /* control whether db file extens GBLREF uint4 ydbDebugLevel; GBLREF jnlpool_addrs_ptr_t jnlpool; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_DBFILERR); error_def(ERR_DBFILEXT); error_def(ERR_DSKSPACEFLOW); @@ -183,6 +195,14 @@ int4 gdsfilext(block_id blocks, block_id filesize, boolean_t trans_in_prog) - DIVIDE_ROUND_UP(cs_data->trans_hist.total_blks, bplmap) + blocks, bplmap - 1) - DIVIDE_ROUND_UP(cs_data->trans_hist.total_blks, bplmap); new_blocks = blocks + new_bit_maps; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP >= cs_data->trans_hist.total_blks) + && (BLKS_PER_LMAP < (cs_data->trans_hist.total_blks + new_blocks))) + { + new_blocks += (ydb_skip_bml_num - BLKS_PER_LMAP); + new_bit_maps += (ydb_skip_bml_num - BLKS_PER_LMAP) / BLKS_PER_LMAP; + } +# endif assert((0 < new_blocks) || (!cs_data->defer_allocate && (0 == new_blocks))); if (new_blocks + cs_data->trans_hist.total_blks > MAXTOTALBLKS(cs_data)) { @@ -196,6 +216,10 @@ int4 gdsfilext(block_id blocks, block_id filesize, boolean_t trans_in_prog) RTS_ERROR_CSA_ABT(cs_addrs, VARLSTCNT(5) ERR_DBFILERR, 2, DB_LEN_STR(gv_cur_region), save_errno); } else { +# ifdef DEBUG + if (0 == ydb_skip_bml_num) + { +# endif if (!(ydbDebugLevel & GDL_IgnoreAvailSpace)) { /* Bypass this space check if debug flag above is on. Allows us to create a large sparce DB * in space it could never fit it if wasn't sparse. Needed for some tests. @@ -218,6 +242,9 @@ int4 gdsfilext(block_id blocks, block_id filesize, boolean_t trans_in_prog) } } } +# ifdef DEBUG + } +# endif } # ifdef DEBUG if ( (WBTEST_ENABLED(WBTEST_MM_CONCURRENT_FILE_EXTEND) && dollar_tlevel && !MEMCMP_LIT(gv_cur_region->rname, "DEFAULT")) || @@ -482,6 +509,11 @@ int4 gdsfilext(block_id blocks, block_id filesize, boolean_t trans_in_prog) for (map = ROUND_UP(old_total, bplmap); map < new_total; map += bplmap) { DEBUG_ONLY(new_bit_maps--;) +# ifdef DEBUG + assert(0 < map); + if ((0 != ydb_skip_bml_num) && (map < ydb_skip_bml_num)) + continue; +# endif if (SS_NORMAL != (status = bml_init(map))) { GDSFILEXT_CLNUP; @@ -519,6 +551,11 @@ int4 gdsfilext(block_id blocks, block_id filesize, boolean_t trans_in_prog) assert(cs_data->blks_to_upgrd == (inctn_detail.blks2upgrd_struct.blks_to_upgrd_delta + prev_extend_blks_to_upgrd)); assert((0 < blocks) || (!cs_data->defer_allocate && (0 == new_blocks))); assert(0 < (cs_addrs->ti->free_blocks + blocks)); +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP >= cs_data->trans_hist.total_blks) + && (BLKS_PER_LMAP < (cs_data->trans_hist.total_blks + new_blocks))) + blocks += (ydb_skip_bml_num - BLKS_PER_LMAP) / BLKS_PER_LMAP * (BLKS_PER_LMAP - 1); +# endif cs_addrs->ti->free_blocks += blocks; cs_addrs->total_blks = cs_addrs->ti->total_blks = new_total; blocks = old_total; diff --git a/sr_unix/gdsfilext_nojnl.c b/sr_unix/gdsfilext_nojnl.c index 2abb309d3..f3acc8ecd 100644 --- a/sr_unix/gdsfilext_nojnl.c +++ b/sr_unix/gdsfilext_nojnl.c @@ -3,7 +3,7 @@ * Copyright (c) 2012-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2019 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2019-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -62,6 +62,10 @@ GBLREF jnl_gbls_t jgbl; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + error_def(ERR_DBFILERR); /* Minimal file extend. Called (at the moment) from mur_back_process.c when processing JRT_TRUNC record. @@ -126,6 +130,11 @@ int gdsfilext_nojnl(gd_region* reg, block_id new_total, block_id old_total) /* initialize bitmaps, if any new ones are added */ for (ii = ROUND_UP(old_total, BLKS_PER_LMAP); ii < new_total; ii += BLKS_PER_LMAP) { +# ifdef DEBUG + assert(0 < ii); + if ((0 != ydb_skip_bml_num) && (ii < ydb_skip_bml_num)) + continue; +# endif offset = (off_t)BLK_ZERO_OFF(csd->start_vbn) + (off_t)ii * blk_size; DB_LSEEKWRITE(csa, udi, udi->fn, udi->fd, offset, aligned_buff, blk_size, status); if (0 != status) diff --git a/sr_unix/gtmio.h b/sr_unix/gtmio.h index a5b3d4c4a..1101e8174 100644 --- a/sr_unix/gtmio.h +++ b/sr_unix/gtmio.h @@ -3,7 +3,7 @@ * Copyright (c) 2001-2018 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2018-2021 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -384,10 +384,19 @@ MBSTART { \ RC = -1; /* Something kept us from reading what we wanted */ \ } MBEND -#define DB_LSEEKREAD(UDI, FD, OFFSET, BUFF, SIZE, STATUS) \ -MBSTART { \ - DBG_CHECK_DIO_ALIGNMENT(UDI, OFFSET, BUFF, SIZE); \ - LSEEKREAD(FD, OFFSET, BUFF, SIZE, STATUS); \ +#ifdef DEBUG +GBLREF gtm_int8 ydb_skip_bml_num; /* Use "gtm_int8" since it is possible "block_id" type is not yet defined */ +#endif + +#define DB_LSEEKREAD(UDI, FD, OFFSET, BUFF, SIZE, STATUS) \ +MBSTART { \ + DBG_CHECK_DIO_ALIGNMENT(UDI, OFFSET, BUFF, SIZE); \ + assert((0 == ydb_skip_bml_num) || (0 == OFFSET) || (NULL == ((unix_db_info *)UDI)->s_addrs.hdr) \ + || ((BLK_ZERO_OFF(((unix_db_info *)UDI)->s_addrs.hdr->start_vbn) \ + + ((gtm_int8)BLKS_PER_LMAP * ((unix_db_info *)UDI)->s_addrs.hdr->blk_size)) > OFFSET) \ + || ((BLK_ZERO_OFF(((unix_db_info *)UDI)->s_addrs.hdr->start_vbn) \ + + (ydb_skip_bml_num * ((unix_db_info *)UDI)->s_addrs.hdr->blk_size)) <= OFFSET)); \ + LSEEKREAD(FD, OFFSET, BUFF, SIZE, STATUS); \ } MBEND /* The below macro is almost the same as LSEEKREAD except it has an extra parameter where the number of diff --git a/sr_unix/mu_cre_file.c b/sr_unix/mu_cre_file.c index 25cefdc24..31eb1a12f 100644 --- a/sr_unix/mu_cre_file.c +++ b/sr_unix/mu_cre_file.c @@ -3,7 +3,7 @@ * Copyright (c) 2001-2021 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2018-2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2018-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -128,6 +128,7 @@ GBLREF sgmnt_data_ptr_t cs_data; GBLREF uint4 ydbDebugLevel; #ifdef DEBUG GBLREF boolean_t in_mu_cre_file; +GBLREF block_id ydb_skip_bml_num; #endif STATICDEF int mu_cre_file_fd; /* needed for "mu_cre_file_ch" */ @@ -403,6 +404,10 @@ unsigned char mu_cre_file(boolean_t caller_is_mupip_create) * manner as every non-bitmap block must have an associated bitmap) */ cs_data->trans_hist.total_blks += DIVIDE_ROUND_UP(cs_data->trans_hist.total_blks, BLKS_PER_LMAP - 1); +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP < cs_data->trans_hist.total_blks)) + cs_data->trans_hist.total_blks += (ydb_skip_bml_num - BLKS_PER_LMAP); +# endif cs_data->extension_size = gv_cur_region->dyn.addr->ext_blk_count; /* Check if this file is an encrypted database. If yes, do init */ if (IS_ENCRYPTED(gv_cur_region->dyn.addr->is_encrypted)) @@ -454,6 +459,7 @@ unsigned char mu_cre_file(boolean_t caller_is_mupip_create) } if (!cs_data->defer_allocate) { + assert(0 == ydb_skip_bml_num); status = posix_fallocate(udi->fd, 0, BLK_ZERO_OFF(cs_data->start_vbn) + ((off_t)(cs_data->trans_hist.total_blks + 1) * cs_data->blk_size)); if (0 != status) diff --git a/sr_unix/mu_swap_root.c b/sr_unix/mu_swap_root.c index 60c1ff4f2..d78ae1f81 100644 --- a/sr_unix/mu_swap_root.c +++ b/sr_unix/mu_swap_root.c @@ -79,6 +79,10 @@ GBLREF unsigned char cw_set_depth; GBLREF unsigned char rdfail_detail; GBLREF unsigned int t_tries; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + #define RETRY_SWAP (0) #define ABORT_SWAP (1) @@ -306,6 +310,17 @@ block_id swap_root_or_directory_block(int parent_blk_lvl, int child_blk_lvl, src t_abort(gv_cur_region, csa); return ABORT_SWAP; } +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (1 == master_bit)) + { + master_bit = bmm_find_free(ydb_skip_bml_num / BLKS_PER_LMAP, csa->bmm, num_local_maps); + if ((NO_FREE_SPACE == master_bit) || (1 == master_bit)) + { + t_abort(gv_cur_region, csa); + return ABORT_SWAP; + } + } +# endif bmlhist.blk_num = master_bit * BLKS_PER_LMAP; if (NULL == (bmlhist.buffaddr = t_qread(bmlhist.blk_num, (sm_int_ptr_t)&bmlhist.cycle, &bmlhist.cr))) { diff --git a/sr_unix/mu_truncate.c b/sr_unix/mu_truncate.c index 187f226f4..9582aefb3 100644 --- a/sr_unix/mu_truncate.c +++ b/sr_unix/mu_truncate.c @@ -3,7 +3,7 @@ * Copyright (c) 2012-2021 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2019-2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2019-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -213,6 +213,10 @@ boolean_t mu_truncate(int4 truncate_percent, mval *keep_mval) return TRUE; } lmap_blk_num = lmap_num * BLKS_PER_LMAP; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= lmap_blk_num) && (lmap_blk_num < ydb_skip_bml_num)) + break; +# endif if (csa->nl->highest_lbm_with_busy_blk >= lmap_blk_num) { found_busy_blk = lmap_blk_num; diff --git a/sr_unix/mubinccpy.c b/sr_unix/mubinccpy.c index 610f11aac..fa29e9c57 100644 --- a/sr_unix/mubinccpy.c +++ b/sr_unix/mubinccpy.c @@ -3,7 +3,7 @@ * Copyright (c) 2001-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2020 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2020-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -333,6 +333,13 @@ bool mubinccpy (backup_reg_list *list) read_offset = (off_t)BLK_ZERO_OFF(header->start_vbn); for (blk_num_base = 0; blk_num_base < header->trans_hist.total_blks; blk_num_base += blks_per_buff) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= blk_num_base) && (blk_num_base < ydb_skip_bml_num)) + { + read_offset += (bsize * (ydb_skip_bml_num - blk_num_base)); + blk_num_base = ydb_skip_bml_num; + } +# endif if (online && (0 != cs_addrs->shmpool_buffer->failed)) break; if ((header->trans_hist.total_blks - blk_num_base) < blks_per_buff) @@ -364,6 +371,10 @@ bool mubinccpy (backup_reg_list *list) for (i = 0; i < blks_per_buff; i++, bptr = (blk_hdr *)((char *)bptr + bsize)) { blk_num = blk_num_base + i; +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (BLKS_PER_LMAP <= blk_num) && (blk_num < ydb_skip_bml_num)) + continue; +# endif if (mu_ctrly_occurred || mu_ctrlc_occurred) { free(outptr); diff --git a/sr_unix/mupip_restore.c b/sr_unix/mupip_restore.c index d603e4b41..04b60a92a 100644 --- a/sr_unix/mupip_restore.c +++ b/sr_unix/mupip_restore.c @@ -3,7 +3,7 @@ * Copyright (c) 2001-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * - * Copyright (c) 2019-2022 YottaDB LLC and/or its subsidiaries. * + * Copyright (c) 2019-2023 YottaDB LLC and/or its subsidiaries. * * All rights reserved. * * * * This source code contains the intellectual property * @@ -81,6 +81,10 @@ GBLREF uint4 pipe_child; GBLREF gd_region *gv_cur_region; GBLREF int4 restore_read_errno; +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + LITREF char *gtm_dbversion_table[]; LITREF char *mdb_ver_names[]; @@ -213,6 +217,11 @@ void mupip_restore(void) } if (udi->fd_opened_with_o_direct) memcpy(&old_data, csd, SGMNT_HDR_LEN); +# ifdef DEBUG + udi->s_addrs.hdr = &old_data; /* Now that we have read in the file header, point udi to it for the next dbfilop. + * This is used by the DB_LSEEKREAD macro if "ydb_skip_bml_num" is non-zero. + */ +# endif if (memcmp(old_data.label, label, GDS_LABEL_SZ) && memcmp(old_data.label, v6_label, GDS_LABEL_SZ)) { util_out_print("Output file !AD has an unrecognizable format", TRUE, n_len, db_name); @@ -421,6 +430,11 @@ void mupip_restore(void) */ new_eof = (off_t)BLK_ZERO_OFF(old_start_vbn) + ((off_t)inhead.db_total_blks * old_blk_size); new_size = new_eof + old_blk_size; +# ifdef DEBUG + udi->s_addrs.hdr = csd; /* Point udi to an up to date file header as it is used by the below + * "db_write_eof_block()" call if "ydb_skip_bml_num" is non-zero. + */ +# endif status = db_write_eof_block(udi, db_fd, old_blk_size, new_eof, &(TREF(dio_buff))); if (0 != status) { @@ -496,6 +510,10 @@ void mupip_restore(void) bml_newmap((blk_hdr_ptr_t)newmap, BM_SIZE(bplmap), curr_tn, csd->desired_db_format); for (ii = ROUND_UP(old_tot_blks, bplmap); ii < inhead.db_total_blks; ii += bplmap) { +# ifdef DEBUG + if ((0 != ydb_skip_bml_num) && (0 < ii) && (ii < ydb_skip_bml_num)) + ii = ydb_skip_bml_num; +# endif offset = (off_t)BLK_ZERO_OFF(old_start_vbn) + (off_t)ii * old_blk_size; DB_LSEEKWRITE(NULL, udi, NULL, db_fd, offset, newmap, old_blk_size, status); if (0 != status) diff --git a/sr_unix/warn_db_sz.c b/sr_unix/warn_db_sz.c index ecdeecd15..c29087503 100644 --- a/sr_unix/warn_db_sz.c +++ b/sr_unix/warn_db_sz.c @@ -3,6 +3,9 @@ * Copyright (c) 2018-2020 Fidelity National Information * * Services, Inc. and/or its subsidiaries. All rights reserved. * * * + * Copyright (c) 2023 YottaDB LLC and/or its subsidiaries. * + * All rights reserved. * + * * * This source code contains the intellectual property * * of its copyright holder(s), and is made available * * under a license. If you do not know the terms of * @@ -22,12 +25,20 @@ error_def(ERR_LOWSPC); +#ifdef DEBUG +GBLREF block_id ydb_skip_bml_num; +#endif + void warn_db_sz(char *db_fname, block_id prev_blocks, block_id curr_blocks, block_id tot_blocks) { double new_sz_frac; double old_sz_frac; int diff; +# ifdef DEBUG + if (0 != ydb_skip_bml_num) + return; /* Do not issue LOWSPC warning if huge sparse dbs are possible */ +# endif new_sz_frac = (WBTEST_ENABLED(WBTEST_DB_BLOCKS_WARN)) ? SPCWARNTHRESHOLD : (((double)curr_blocks) / ((double)tot_blocks)) * 100; if (new_sz_frac < SPCWARNTHRESHOLD)