feat(oracle): add the ability to extract historical data for Statspack extracts

scripts/collector/oracle/collect-data.sh

@@ -112,19 +112,20 @@ connectString="$1"
 OpVersion=$2
 DiagPack=$(echo $3 | tr [[:upper:]] [[:lower:]])
 manualUniqueId="${4}"
-statsWindow=${5}
+statsWindow="${5}"
+statsStartDate="${6}"
+statsDBID="${7}"
 
 if ! [ -x "$(command -v ${SQLPLUS})" ]; then
   echo "Could not find ${SQLPLUS} command. Source in environment and try again"
   echo "Exiting..."
   exit 1
 fi
 
-
 ${SQLPLUS} -s /nolog << EOF
 SET DEFINE OFF
 connect ${connectString}
-@${SQL_DIR}/op_collect.sql ${OpVersion} ${SQL_DIR} ${DiagPack} ${V_TAG} ${SQLOUTPUT_DIR} "${manualUniqueId}" ${statsWindow}
+@${SQL_DIR}/op_collect.sql ${OpVersion} ${SQL_DIR} ${DiagPack} ${V_TAG} ${SQLOUTPUT_DIR} "${manualUniqueId}" "${statsWindow}" "${statsStartDate}" "${statsDBID}"
 exit;
 EOF
 
@@ -240,6 +241,10 @@ fi
 
 if [ -f $OUTFILE ]
 then
+  if [ -f opdb__eoj__${V_FILE_TAG}.csv ]
+    then
+      cat opdb__eoj__${V_FILE_TAG}.csv
+  fi
   rm  opdb*${V_FILE_TAG}.csv opdb*${V_FILE_TAG}*.log opdb*${V_FILE_TAG}*.txt
 fi
 
@@ -325,6 +330,8 @@ statsSrc=""
 connStr=""
 manualUniqueId=""
 statsWindow=30
+statsStartDate=""
+statsDBID=""
 
  if [[ $(($# & 1)) == 1 ]] ;
  then
@@ -344,6 +351,8 @@ statsWindow=30
 	 elif [[ "$1" == "--connectionStr" ]];      then connStr="${2}"
 	 elif [[ "$1" == "--manualUniqueId" ]];     then manualUniqueId="${2}"
 	 elif [[ "$1" == "--statsWindow" ]];        then statsWindow="${2}"
+	 elif [[ "$1" == "--statsStartDate" ]];     then statsStartDate="${2}"   # Experimental - Statspack only
+	 elif [[ "$1" == "--statsDBID" ]];          then statsDBID="${2}"        # Experimental - Statspack only
 	 else
 		 echo "Unknown parameter ${1}"
 		 printUsage
@@ -359,16 +368,27 @@ statsWindow=30
 
  if [[ "${statsSrc}" = "awr" ]]; then
           DIAGPACKACCESS="UseDiagnostics"
+	  if [[ "$statsStartDate" != "" ]]; then
+		  echo Parameter statsStartDate is not yet supported with AWR source.
+		  exit 1
+	  fi
+	  if [[ "$statsDBID" != "" ]]; then
+		  echo Parameter statsDBID is not yet supported with AWR source.
+		  exit 1
+	  fi
  elif [[ "${statsSrc}" = "statspack" ]] ; then
           DIAGPACKACCESS="NoDiagnostics"
- else
-	 echo No performance data will be collected.
+ else 
+         echo User requested no performance data collection.
          DIAGPACKACCESS="nostatspack"
  fi
 
- if [[ ${statsWindow} -ne 30 ]] && [[ ${statsWindow} -ne 7 ]] ; then
+ if [[ "${statsWindow}" == "" ]] ; then 
 	 statsWindow=30
  fi
+# if [[ ${statsWindow} -ne 30 ]] && [[ ${statsWindow} -ne 7 ]] ; then
+#        statsWindow=30
+# fi
 
  if [[ "${connStr}" == "" ]] ; then
 	 if [[ "${hostName}" != "" && "${port}" != "" && "${databaseService}" != "" && "${collectionUserName}" != "" && "${collectionUserPass}" != "" ]] ; then
@@ -405,6 +425,16 @@ statsWindow=30
 # MAIN
 #############################################################################
 
+extractorVersion="$(getVersion)"
+
+echo ""
+echo "==================================================================================="
+echo "Database Migration Assessment Database Assessment Collector Version ${OpVersion}"
+printExtractorVersion "${extractorVersion}"
+echo "==================================================================================="
+echo ""
+echo "Connecting to database..."
+
 connectString="${connStr}"
 sqlcmd_result=$(checkVersion "${connectString}" "${OpVersion}" | $GREP DMAFILETAG | cut -d '~' -f 2)
 if [[ "${sqlcmd_result}" = "" ]];
@@ -417,14 +447,6 @@ retval=$?
 
 # DIAGPACKACCESS="$2"
 
-extractorVersion="$(getVersion)"
-
-echo ""
-echo "==================================================================================="
-echo "Database Migration Assessment Database Assessment Collector Version ${OpVersion}"
-printExtractorVersion "${extractorVersion}"
-echo "==================================================================================="
-
 if [ $retval -eq 0 ]; then
   if [ "$(echo ${sqlcmd_result} | $GREP -E '(ORA-|SP2-)')" != "" ]; then
     echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"
@@ -445,7 +467,7 @@ if [ $retval -eq 0 ]; then
       fi
     fi
     V_TAG="$(echo ${sqlcmd_result} | cut -d '|' -f2).csv"; export V_TAG
-    executeOP "${connectString}" ${OpVersion} ${DIAGPACKACCESS} "${manualUniqueId}" $statsWindow
+    executeOP "${connectString}" ${OpVersion} ${DIAGPACKACCESS} "${manualUniqueId}" "$statsWindow" "$statsStartDate" "$statsDBID"
     retval=$?
     if [ $retval -ne 0 ]; then
       createErrorLog  $(echo ${V_TAG} | sed 's/.csv//g')

scripts/collector/oracle/sql/extracts/archlogs.sql

@@ -24,7 +24,7 @@ SELECT :v_pkey AS pkey,
        round(sum(blocks * block_size)/1024/1024) as mbytes,
        :v_dma_source_id AS DMA_SOURCE_ID, :v_manual_unique_id AS DMA_MANUAL_ID
 FROM gv$archived_log
-WHERE first_time >= trunc(sysdate) - '&&dtrange'
+WHERE first_time >= trunc(sysdate) - '&&dtrange' 
 GROUP BY trunc(first_time), thread#, to_char(first_time, 'HH24'), dest_id
 ;
 spool off

scripts/collector/oracle/sql/extracts/backups.sql

@@ -24,7 +24,7 @@ SELECT :v_pkey AS pkey,
        round(sum(output_bytes)/1024/1024) AS mbytes_out,
        :v_dma_source_id AS DMA_SOURCE_ID, :v_manual_unique_id AS DMA_MANUAL_ID
 FROM v$rman_backup_job_details a
-WHERE start_time >= trunc(sysdate) - '&&dtrange'
+WHERE start_time >= trunc(sysdate) - '&&dtrange' 
 GROUP BY trunc(start_time), input_type, &v_a_con_id
 ;
 spool off

scripts/collector/oracle/sql/extracts/dbsummary.sql

@@ -38,7 +38,7 @@ SELECT :v_pkey AS pkey,
         FROM   (SELECT TRUNC(first_time) dia,
                        COUNT(*)          conta
                 FROM   v$log_history
-                WHERE  first_time >= TRUNC(SYSDATE) - '&&dtrange'
+                WHERE  first_time >= TRUNC(SYSDATE) - '&&dtrange'   
                        AND first_time < TRUNC(SYSDATE)
                 GROUP  BY TRUNC(first_time)),
                v$log)                                                           AS redo_gb_per_day,

scripts/collector/oracle/sql/extracts/defines.sql

@@ -41,16 +41,19 @@ prompt v_c_con_id         = &v_c_con_id
 prompt v_h_con_id         = &v_h_con_id
 prompt v_data_type_exp    = &v_data_type_exp
 prompt v_db_container_col = &v_db_container_col
-prompt v_dbid             = &v_dbid
-prompt v_dma_source_id    = &v_dma_source_id
-prompt v_dbname           = &v_dbname
-prompt v_dbparam_dflt_col = &v_dbparam_dflt_col
-prompt v_dbversion        = &v_dbversion
-prompt v_dopluggable      = &v_dopluggable
-prompt v_editionable_col  = &v_editionable_col
-prompt v_hora             = &v_hora
-prompt v_host             = &v_host
-prompt v_inst             = &v_inst
+prompt v_dbid             = &v_dbid 
+prompt v_statsDBIDRequested = &v_statsDBIDRequested
+prompt v_statsDBID        = &v_statsDBID
+prompt v_dbname           = &v_dbname 
+prompt v_dbparam_dflt_col = &v_dbparam_dflt_col 
+prompt v_dbversion        = &v_dbversion 
+prompt v_dodiagnostics    = &v_dodiagnostics
+prompt v_dopluggable      = &v_dopluggable 
+prompt v_editionable_col  = &v_editionable_col 
+prompt v_hora             = &v_hora 
+prompt v_host             = &v_host 
+prompt v_info_prompt      = &p_info_prompt
+prompt v_inst             = &v_inst 
 prompt v_io_function_sql  = &v_io_function_sql
 prompt v_is_container     = &v_is_container
 prompt v_max_snapid       = &v_max_snapid
@@ -60,4 +63,8 @@ prompt v_umf_test         = &v_umf_test
 prompt Applies to STATSPACK collections only:
 prompt v_max_snaptime     = &v_max_snaptime
 prompt v_min_snaptime     = &v_min_snaptime
+prompt v_statsStartDate   = &v_statsStartDate
+prompt v_statsWindow      = &v_statsWindow
+prompt
+SELECT 'v_dma_sourceid     = ' || :v_dma_source_id FROM DUAL;
 spool off

scripts/collector/oracle/sql/extracts/eoj.sql

@@ -16,4 +16,5 @@
 -- This file intentionally has no extraction SQL.
 spool &outputdir/opdb__eoj__&v_tag
 prompt END_OF_DMA_COLLECTION
+prompt &p_end_of_job_summary
 spool off

scripts/collector/oracle/sql/extracts/statspack/awrhistcmdtypes.sql

@@ -21,8 +21,8 @@ prompt PKEY|CON_ID|HH|COMMAND_TYPE|CNT|AVG_BUFFER_GETS|AVG_ELASPED_TIME|AVG_ROWS
 WITH vcmdtype AS(
 SELECT :v_pkey AS pkey,
        'N/A'                       AS con_id,
-       TO_CHAR(sn.snap_time, 'hh24')     AS hh24,
-       ss.command_type,
+       TO_CHAR(snap_time, 'hh24')     AS hh24,
+       command_type, 
        COUNT(1)                          AS cnt,
        ROUND(AVG(delta_buffer_gets))           AS avg_buffer_gets,
        ROUND(AVG(delta_elapsed_time))          AS avg_elapsed_time,
@@ -34,8 +34,10 @@ SELECT :v_pkey AS pkey,
        ROUND(AVG(delta_application_wait_time)) AS avg_apwait,
        ROUND(AVG(delta_concurrency_wait_time)) AS avg_ccwait,
        ROUND(AVG(delta_plsql_exec_time))       AS avg_plsexec_time,
-       aa.name                           AS command_name
-FROM
+       name                                    AS command_name
+FROM 
+( SELECT ss.*, aa.name, sn.snap_time, sn.startup_time, sn.lag_startup_time 
+  FROM
 (
 select snap_id, dbid, instance_number, text_subset, old_hash_value, command_type, force_matching_signature, sql_id,
 s.executions,
@@ -160,15 +162,20 @@ s.executions,
       0) AS delta_java_exec_time
 From STATS$SQL_SUMMARY s
      ) ss
-    JOIN stats$snapshot sn
+    JOIN ( SELECT dbid, instance_number, snap_id, snap_time, startup_time, lag(startup_time) OVER (PARTITION BY dbid, instance_number ORDER BY snap_time) AS lag_startup_time 
+	   FROM stats$snapshot 
+	   WHERE snap_time BETWEEN '&&v_min_snaptime' AND '&&v_max_snaptime'
+	   AND dbid = &&v_statsDBID
+         ) sn
       ON     ss.dbid = sn.dbid
          AND ss.snap_id = sn.snap_id
          AND ss.instance_number = sn.instance_number
     LEFT OUTER join audit_actions aa
                  ON ss.command_type = aa.action
-WHERE sn.snap_time BETWEEN '&&v_min_snaptime' AND '&&v_max_snaptime'
+                 )
+    WHERE startup_time = lag_startup_time
 GROUP BY :v_pkey,
-          'N/A' , TO_CHAR(sn.snap_time, 'hh24'),  ss.command_type, aa.name
+          'N/A' , TO_CHAR(snap_time, 'hh24'),  command_type, name
 )
 SELECT pkey , con_id AS sp_con_id, hh24 , command_type , cnt , avg_buffer_gets , avg_elapsed_time ,
        avg_rows_processed , avg_executions , avg_cpu_time , avg_iowait , avg_clwait ,

scripts/collector/oracle/sql/extracts/statspack/awrhistosstat.sql

@@ -57,16 +57,19 @@ WITH v_osstat_all
                        OVER (
                        PARTITION BY s.dbid, s.instance_number, osname.STAT_NAME
                        ORDER BY s.snap_id),
-                    0), 0) AS delta_value
+                    0), 0) AS delta_value,
+                    snap.startup_time,
+                    LAG(snap.startup_time) OVER (PARTITION BY snap.dbid, snap.instance_number, s.osstat_id ORDER BY snap.snap_time) AS lag_startup_time
                     FROM STATS$OSSTAT s
                          inner join STATS$SNAPSHOT snap
                          ON s.snap_id = snap.snap_id
                          AND s.instance_number = snap.instance_number
                          AND s.dbid = snap.dbid
                          inner join STATS$OSSTATNAME osname
                          ON s.osstat_id = osname.osstat_id
-                    WHERE snap.snap_time BETWEEN (SELECT max(snap_time) FROM  STATS$SNAPSHOT WHERE snap_time < '&&v_min_snaptime'  ) AND '&&v_max_snaptime'
-                    AND s.dbid = '&&v_dbid') os ) ,
+                    WHERE snap.snap_time BETWEEN  '&&v_min_snaptime' AND '&&v_max_snaptime'
+                    AND s.dbid = '&&v_statsDBID') os
+              WHERE startup_time = lag_startup_time) ,
 vossummary AS (
 SELECT :v_pkey AS pkey,
        dbid,

scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetrichist.sql

@@ -19,28 +19,30 @@ spool &outputdir/opdb__awrhistsysmetrichist__&v_tag
 prompt PKEY|DBID|INSTANCE_NUMBER|HOUR|METRIC_NAME|METRIC_UNIT|AVG_VALUE|MODE_VALUE|MEDIAN_VALUE|MIN_VALUE|MAX_VALUE|SUM_VALUE|PERC50|PERC75|PERC90|PERC95|PERC100|DMA_SOURCE_ID|DMA_MANUAL_ID
 WITH vsysmetric AS (
 SELECT :v_pkey AS pkey,
-       hsm.dbid,
-       hsm.instance_number,
-       TO_CHAR(dhsnap.snap_time, 'hh24')          hour,
-       hsm.name as metric_name,
+       dbid,
+       instance_number,
+       TO_CHAR(snap_time, 'hh24')          hour,
+       name as metric_name,
        null as metric_unit,
-       ROUND(AVG(hsm.delta_value))                           avg_value,
-       ROUND(STATS_MODE(hsm.delta_value))                    mode_value,
-       ROUND(MEDIAN(hsm.delta_value))                        median_value,
-       ROUND(MIN(hsm.delta_value))                           min_value,
-       ROUND(MAX(hsm.delta_value))                           max_value,
-       ROUND(SUM(hsm.delta_value))                           sum_value,
+       ROUND(AVG(delta_value))                           avg_value,
+       ROUND(STATS_MODE(delta_value))                    mode_value,
+       ROUND(MEDIAN(delta_value))                        median_value,
+       ROUND(MIN(delta_value))                           min_value,
+       ROUND(MAX(delta_value))                           max_value,
+       ROUND(SUM(delta_value))                           sum_value,
        ROUND(PERCENTILE_CONT(0.5)
-         within GROUP (ORDER BY hsm.delta_value DESC)) AS "PERC50",
+         within GROUP (ORDER BY delta_value DESC)) AS "PERC50",
        ROUND(PERCENTILE_CONT(0.25)
-         within GROUP (ORDER BY hsm.delta_value DESC)) AS "PERC75",
+         within GROUP (ORDER BY delta_value DESC)) AS "PERC75",
        ROUND(PERCENTILE_CONT(0.10)
-         within GROUP (ORDER BY hsm.delta_value DESC)) AS "PERC90",
+         within GROUP (ORDER BY delta_value DESC)) AS "PERC90",
        ROUND(PERCENTILE_CONT(0.05)
-         within GROUP (ORDER BY hsm.delta_value DESC)) AS "PERC95",
+         within GROUP (ORDER BY delta_value DESC)) AS "PERC95",
        ROUND(PERCENTILE_CONT(0)
-         within GROUP (ORDER BY hsm.delta_value DESC)) AS "PERC100"
-FROM   (
+         within GROUP (ORDER BY delta_value DESC)) AS "PERC100"
+FROM
+( SELECT hsm.*, dhsnap.snap_time, dhsnap.startup_time, dhsnap.lag_startup_time
+  FROM (   (
         SELECT s.snap_id, s.dbid, s.instance_number, s.name, s.value,
                NVL(
                    DECODE(
@@ -50,25 +52,28 @@ FROM   (
                         0),
                  0) AS delta_value
            FROM perfstat.stats$sysstat s ) hsm
-       INNER JOIN stats$snapshot dhsnap
+       INNER JOIN ( SELECT dbid, instance_number, snap_id, snap_time, startup_time, lag(startup_time) OVER (PARTITION BY dbid, instance_number ORDER BY snap_time) AS lag_startup_time
+                    FROM stats$snapshot
+                    WHERE snap_time BETWEEN '&&v_min_snaptime' AND '&&v_max_snaptime'
+                    AND dbid = &&v_statsDBID
+               ) dhsnap
                ON hsm.snap_id = dhsnap.snap_id
                   AND hsm.instance_number = dhsnap.instance_number
-                  AND hsm.dbid = dhsnap.dbid
-WHERE  dhsnap.snap_time BETWEEN '&&v_min_snaptime' AND '&&v_max_snaptime'
-AND hsm.dbid = &&v_dbid
+                  AND hsm.dbid = dhsnap.dbid)
+WHERE startup_time = lag_startup_time)
 GROUP  BY :v_pkey,
-          hsm.dbid,
-          hsm.instance_number,
-          TO_CHAR(dhsnap.snap_time, 'hh24'),
-          hsm.name
-ORDER  BY hsm.dbid,
-          hsm.instance_number,
-          hsm.name,
-          TO_CHAR(dhsnap.snap_time, 'hh24'))
+          dbid,
+          instance_number,
+          TO_CHAR(snap_time, 'hh24'),
+          name
+ORDER  BY dbid,
+          instance_number,
+          name,
+          TO_CHAR(snap_time, 'hh24'))
 SELECT pkey , dbid , instance_number , hour , metric_name ,
        metric_unit , avg_value , mode_value , median_value , min_value , max_value ,
-	   sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100,
-	       :v_dma_source_id AS DMA_SOURCE_ID, :v_manual_unique_id AS DMA_MANUAL_ID
+        sum_value , PERC50 , PERC75 , PERC90 , PERC95 , PERC100,
+            :v_dma_source_id AS DMA_SOURCE_ID, :v_manual_unique_id AS DMA_MANUAL_ID
 FROM vsysmetric;
 spool off
 COLUMN HOUR CLEAR

scripts/collector/oracle/sql/extracts/statspack/awrhistsysmetricsumm.sql

@@ -50,7 +50,9 @@ SELECT :v_pkey AS pkey,
        --hsm.AVERAGE+(2* CASE WHEN ( standard_deviation > (.999999999999999999999999999) AND standard_deviation < 1 )
        --                      AND ( MINVAL = 0 AND AVERAGE = MAXVAL )  then 0 else standard_deviation end ) "PERC95",
        AVG(value) OVER (PARTITION BY hsm.dbid, hsm.instance_number,  TO_CHAR(dhsnap.snap_time, 'hh24')  , hsm.name) + (2 * STDDEV (value)  OVER (PARTITION BY hsm.dbid, hsm.instance_number,  TO_CHAR(dhsnap.snap_time, 'hh24')  , hsm.name)) AS "PERC95",
-       NULL                                   "PERC100"
+       NULL                                   "PERC100",
+       startup_time,
+       lag_startup_time
 FROM (
       SELECT s.snap_id, s.dbid, s.instance_number, s.name, s.value,
              NVL(
@@ -61,12 +63,16 @@ FROM (
                        0),
                 0) AS delta_value
        FROM perfstat.stats$sysstat s )   hsm
-       INNER JOIN stats$snapshot dhsnap
+       INNER JOIN (SELECT dbid, instance_number, snap_id, snap_time, startup_time, 
+	                  lag(startup_time) OVER (PARTITION BY dbid, instance_number ORDER BY snap_time) AS lag_startup_time
+                   FROM stats$snapshot
+		   WHERE snap_time BETWEEN '&&v_min_snaptime' AND '&&v_max_snaptime'
+		   AND dbid = &&v_statsDBID
+                  ) dhsnap
                ON hsm.snap_id = dhsnap.snap_id
                   AND hsm.instance_number = dhsnap.instance_number
                   AND hsm.dbid = dhsnap.dbid
-WHERE  dhsnap.snap_time BETWEEN '&&v_min_snaptime' AND '&&v_max_snaptime'
-AND hsm.dbid = &&v_dbid),
+     ),
 vsysmetricsummperhour as (
     SELECT pkey,
        hsm.dbid,
@@ -91,6 +97,7 @@ vsysmetricsummperhour as (
        ROUND(PERCENTILE_CONT(0)
          within GROUP (ORDER BY hsm.PERC95 DESC)) AS "PERC100"
     FROM vsysmetricsumm hsm
+    WHERE startup_time = lag_startup_time
     GROUP  BY pkey,
             hsm.dbid,
             hsm.instance_number,