Tuesday, July 9, 2024

RAC

 Oracle RAC 12cR1 introduced a new architecture option called Flex Clusters.

blacklist {
        devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st|sdao)[0-9]*"
        devnode "^asm/*"
        devnode "^ofsctl"
        wwid    3600605b0033286b0162b06912bb7c0dd
        wwid    3600605b0033286b0162b06912bb7181b
        wwid    3600605b0033286b0162b06912bb76c7b
}


[root@nmsis2d11 oracleasm]# cat /etc/multipath.conf | grep -A19 defaults
defaults {
        polling_interval 5
        path_grouping_policy multibus
        failback immediate
        user_friendly_names yes
        max_fds 8192
}
blacklist {
        devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st|sdao)[0-9]*"
        devnode "^asm/*"
        devnode "^ofsctl"
        wwid    3600605b0033286b0162b06912bb7c0dd
        wwid    3600605b0033286b0162b06912bb7181b
        wwid    3600605b0033286b0162b06912bb76c7b
}

[root@ipagt1d5 ~]# cat /etc/multipath.conf | grep -A19 defaults
defaults {
        user_friendly_names yes
        failback immediate
        prio const
        find_multipaths yes
        detect_prio yes
        polling_interval 10
}
blacklist {
        devnode "^(ram|raw|loop|fd|md|dm-|sr|scd|st)[0-9]*"
        devnode "^asm/*"
        devnode "^ofsctl"
}

[root@nmsis2d11 oragrid_s2d11_vg]# df -h /dev/mapper/t2ls2idb_s2d11_vg-t2ls2idb
Filesystem                              Size  Used Avail Use% Mounted on
/dev/mapper/t2ls2idb_s2d11_vg-t2ls2idb   99G   60G   34G  64% /opt/app/t2ls2idb
[root@nmsis2d11 oragrid_s2d11_vg]# df -h /dev/mapper/oragrid_s2d11_vg-oragrid
Filesystem                            Size  Used Avail Use% Mounted on
/dev/mapper/oragrid_s2d11_vg-oragrid   99G   56G   38G  60% /opt/app/oragrid
[root@nmsis2d11 oragrid_s2d11_vg]# ls -ltr /dev/mapper/
total 0
crw------- 1 root root 10, 236 Jul  3 14:34 control
lrwxrwxrwx 1 root root       8 Jul 15 02:28 oragrid_s2d11_vg-oragrid -> ../dm-24
lrwxrwxrwx 1 root root       8 Jul 15 02:28 t2ls2idb_s2d11_vg-t2ls2idb -> ../dm-25


[root@nmsis2d11 oragrid_s2d11_vg]# pwd
/dev/oragrid_s2d11_vg
[root@nmsis2d11 oragrid_s2d11_vg]# ls -ltr
total 0
lrwxrwxrwx 1 root root 8 Jul 15 02:28 oragrid -> ../dm-24
[root@nmsis2d11 t2ls2idb_s2d11_vg]# pwd
/dev/t2ls2idb_s2d11_vg
[root@nmsis2d11 t2ls2idb_s2d11_vg]# ls -ltr
total 0
lrwxrwxrwx 1 root root 8 Jul 15 02:28 t2ls2idb -> ../dm-25


[root@nmsis2d11 mapper]# ls -l
total 0
lrwxrwxrwx 1 root root       8 Jul 15 15:29 ASM_20171220_4.0G_EMC_1090_01A3 -> ../dm-22
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_4.0G_EMC_1090_01A4 -> ../dm-1
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_50G_EMC_1090_01A5 -> ../dm-2
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_50G_EMC_1090_01A6 -> ../dm-3
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_50G_EMC_1090_01A7 -> ../dm-4
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_50G_EMC_1090_01A8 -> ../dm-5
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_50G_EMC_1090_01A9 -> ../dm-6
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_50G_EMC_1090_01AA -> ../dm-7
lrwxrwxrwx 1 root root       7 Jul 15 15:29 ASM_20171220_725G_EMC_1090_019B -> ../dm-0
lrwxrwxrwx 1 root root       8 Jul 15 15:29 ASM_20171220_725G_EMC_1090_019C -> ../dm-11
lrwxrwxrwx 1 root root       8 Jul 15 16:27 ASM_20171220_725G_EMC_1090_019D -> ../dm-17
lrwxrwxrwx 1 root root       8 Jul 15 16:27 ASM_20171220_725G_EMC_1090_019E -> ../dm-18
lrwxrwxrwx 1 root root       8 Jul 15 16:00 ASM_20171220_725G_EMC_1090_019F -> ../dm-19
lrwxrwxrwx 1 root root       8 Jul 15 16:00 ASM_20171220_725G_EMC_1090_01A0 -> ../dm-23
lrwxrwxrwx 1 root root       8 Jul 15 16:25 ASM_20171220_725G_EMC_1090_01A1 -> ../dm-20
lrwxrwxrwx 1 root root       8 Jul 15 15:29 ASM_20171220_725G_EMC_1090_01A2 -> ../dm-21
lrwxrwxrwx 1 root root       8 Jul 15 16:00 ASM_20200311_725G_EMC_1090_0517 -> ../dm-10
lrwxrwxrwx 1 root root       8 Jul 15 16:15 ASM_20200311_725G_EMC_1090_0518 -> ../dm-12
lrwxrwxrwx 1 root root       8 Jul 15 16:00 ASM_20200311_725G_EMC_1090_0519 -> ../dm-13
lrwxrwxrwx 1 root root       8 Jul 15 15:29 ASM_20200311_725G_EMC_1090_051A -> ../dm-14
lrwxrwxrwx 1 root root       8 Jul 15 15:29 ASM_20200311_725G_EMC_1090_051B -> ../dm-15
lrwxrwxrwx 1 root root       8 Jul 15 15:29 ASM_20200311_725G_EMC_1090_051C -> ../dm-16
crw------- 1 root root 10, 236 Jul  3 14:34 control
lrwxrwxrwx 1 root root       7 Jul 15 15:29 LOCAL_20171220_100G_EMC_1090_01AB -> ../dm-8
lrwxrwxrwx 1 root root       7 Jul 15 15:29 LOCAL_20171220_100G_EMC_1090_01AC -> ../dm-9
lrwxrwxrwx 1 root root       8 Jul 15 02:28 oragrid_s2d11_vg-oragrid -> ../dm-24
lrwxrwxrwx 1 root root       8 Jul 15 02:28 t2ls2idb_s2d11_vg-t2ls2idb -> ../dm-25


[root@nmsis2d11 mapper]# df -Th
Filesystem                             Type      Size  Used Avail Use% Mounted on
devtmpfs                               devtmpfs   36G     0   36G   0% /dev
tmpfs                                  tmpfs      36G  389M   35G   2% /dev/shm
tmpfs                                  tmpfs      36G  234M   36G   1% /run
tmpfs                                  tmpfs      36G     0   36G   0% /sys/fs/cgroup
/dev/sda7                              ext4       19G  8.0G  9.3G  47% /
/dev/mapper/t2ls2idb_s2d11_vg-t2ls2idb ext4       99G   60G   34G  64% /opt/app/t2ls2idb
/dev/sda1                              ext4      976M  276M  634M  31% /boot
/dev/sda6                              ext4      4.8G   13M  4.6G   1% /tmp
/dev/sda5                              ext4       15G  2.3G   12G  17% /var
/dev/sda2                              ext4       15G  1.7G   13G  12% /home
/dev/sdd1                              ext4       69G  202M   65G   1% /var/crash
/dev/mapper/oragrid_s2d11_vg-oragrid   ext4       99G   56G   38G  60% /opt/app/oragrid
dy-tdevnas1:/dbbkups01                 nfs       4.0T  3.7T  402G  91% /pac/prod_backup2
tmpfs                                  tmpfs     7.1G     0  7.1G   0% /run/user/51012
tmpfs                                  tmpfs     7.1G     0  7.1G   0% /run/user/0
tmpfs                                  tmpfs     7.1G     0  7.1G   0% /run/user/1563708
dy-tdevnas1.snt.bst.bls.com:/osddba    nfs       200G  184G   17G  92% /nas/osddba
dy-tdevnas1.snt.bst.bls.com:/osd       nfs       2.0T  967G  1.1T  48% /nas/osd
tmpfs                                  tmpfs     7.1G     0  7.1G   0% /run/user/21295

dy-tdevnas1:/dbbkups01     /pac/prod_backup2        nfs     rw,bg,tcp,nfsvers=3,suid,noac,hard,rsize=32768,wsize=32768 0 0

ORA-01103: database name 'IPAGI1DB' in control file is not 'IPAGIT1D'


https://www.pythian.com/blog/technical-track/oracle-rman-restore-to-the-same-machine-as-the-original-database  --> restore database from rman backup with in the same host

ACTION!="add|change", GOTO="dm_end"
ENV{DM_UDEV_RULES_VSN}!="?*", GOTO="dm_end"
# Set permissions for a DM device name starting with ASM for Oracle RAC
ENV{DM_NAME}=="ASM*", OWNER:="oragrid", GROUP:="asmadmin", MODE:="0660"
LABEL="dm_end"

[root@nmsis6d11 rules.d]# ls -l
total 8
-rw-r----- 1 root root 190 Oct  1  2023 55-usm.rules
-rw-r--r-- 1 root root 144 Jun 24 15:23 99-oracle.rules
[root@nmsis6d11 rules.d]# more 99-oracle.rules
# Set permissions for a DM device name starting with ASM for Oracle RAC
ENV{DM_NAME}=="ASM*", OWNER:="oragrid", GROUP:="asmadmin", MODE:="0660"
[root@nmsis6d11 rules.d]#

[root@ipagt5d12 rules.d]# more 99-oracle.rules
ACTION!="add|change", GOTO="dm_end"
ENV{DM_UDEV_RULES_VSN}!="?*", GOTO="dm_end"
# Set permissions for a DM device name starting with ASM for Oracle RAC
ENV{DM_NAME}=="ASM*", OWNER:="oragrid", GROUP:="asmadmin", MODE:="0660"
LABEL="dm_end"



-- Step 1: Check free space in datafiles
SELECT FILE_NAME, BYTES, BLOCKS, FREE_SPACE
FROM DBA_DATA_FILES df
JOIN (SELECT FILE_ID, SUM(BYTES) FREE_SPACE
      FROM DBA_FREE_SPACE
      GROUP BY FILE_ID) fs
ON df.FILE_ID = fs.FILE_ID
WHERE df.TABLESPACE_NAME = 'your_tablespace_name';

-- Step 2: Move segments (tables and indexes) to free up space
ALTER TABLE your_table_name MOVE;
ALTER INDEX your_index_name REBUILD;

-- Step 3: Resize datafiles
ALTER DATABASE DATAFILE 'your_datafile_name1' RESIZE 500M;
ALTER DATABASE DATAFILE 'your_datafile_name2' RESIZE 300M;
-- Repeat for all datafiles

-- Step 4: Coalesce free space
ALTER TABLESPACE your_tablespace_name COALESCE;

 select sum(bytes)/1024/1024/1024 from dba_segments where TABLESPACE_NAME='INDICES';
select sum(bytes)/1024/1024/1024 from dba_data_files where tablespace_name='TOOLS';



DECLARE
    v_id NUMBER := 1;
    v_clob CLOB;
    v_name VARCHAR2(50) := 'Test Data';
    v_date DATE := SYSDATE;
    v_chunk CLOB := RPAD('X', 1000, 'X');  -- 1KB of data
    v_clob_size NUMBER := 10485760;        -- 10MB per row

    PROCEDURE insert_row(p_id IN NUMBER, p_name IN VARCHAR2, p_date IN DATE, p_clob IN CLOB) IS
    BEGIN
        INSERT INTO test_table (id, name, created_date, data)
        VALUES (p_id, p_name, p_date, p_clob);
    END;
BEGIN
    FOR i IN 1..1024 LOOP  -- 1024 * 10MB = 10GB
        DBMS_LOB.CREATETEMPORARY(v_clob, TRUE);
        DBMS_LOB.OPEN(v_clob, DBMS_LOB.LOB_READWRITE);

        FOR j IN 1..(v_clob_size / 1000) LOOP
            DBMS_LOB.WRITEAPPEND(v_clob, LENGTH(v_chunk), v_chunk);
        END LOOP;

        insert_row(v_id, v_name, v_date, v_clob);

        v_id := v_id + 1;
        DBMS_LOB.FREETEMPORARY(v_clob);
    END LOOP;

    COMMIT;
END;
/

From the provided information we see that DATA tablespace is having around 500 GB of free space, but you could not able to shrink all the free space from datafiles.
This is due to fragmentation in the datafiles. There could be an active datablock occupied by the objects data at the end of the datafile which is causing not to shrink the datafile.

To overcome this you need reorg all the objects in the tablespace to be able to shrink more space in the datafiles.

Please find below documents on how to move objects in the tablespace:

How to Move Different Database Objects to Another Tablespace ( Doc ID 1915040.1 )


0----
Do i need to reorg all the objects first , then go for shrinking the datafiles right?

moving within the same tablespace will do the job right?

no need to create new tablespace and then move into that newone --> ?
-------
Yes you need to reorg all the objects first and go for shrinking the datafiles.

No need to create the new tablespace as there is enough free space in the DATA tablespace. Moving within the same tablespace will do the job.
--------

t1ls1jdb@nmsjs1d1(1110) t1ls1jd1 /opt/app/t1ls1jdb/oracle$ more /home/oracle/oraadmin/rman_backup.sh
#!/bin/ksh
#
# ******************************************************************
# *                           NOTICE                               *
# *** NOT FOR USE OR DISCLOSURE OUTSIDE THE AT&T COMPANIES      ****
# ******************************************************************
#          Copyright 2009. AT&T. All rights reserved.
# ******************************************************************
# *
# *   Module name       : rman_backup.sh
# *
# *   Module description: Backup the database using RMAN.
# *
# *   Date created      :
# *
# *   Author            :
# *
# *   Modification History      : Pratul Malhotra
# *   --------------------------------------------------------------
# *
# *******************************************************************
#
#set -x
#
# Format Log File for Backups
#
format_backup_logfile_name()
{
   # Create Backup Log Directory if it does not Exist
   #
   if [ ! -d ${BACKUPSLOG_DIR} ]
   then
      mkdir -p ${BACKUPSLOG_DIR}
   fi

   if [ ${ARCHIVEONLY} = Y ]
   then
      LOGFILE=${BACKUPSLOG_DIR}/DB_Arch_Backup_`date +%Y%m%d%H%M`.log
   elif [ ${INCREMENTALONLY} = Y ]
   then
     LOGFILE=${BACKUPSLOG_DIR}/DB_Incremental_Backup_`date +%Y%m%d%H%M`.log
   else
     LOGFILE=${BACKUPSLOG_DIR}/DB_Full_Backup_`date +%Y%m%d%H%M`.log
   fi
}


#
# List Contents of Backup Directory
#
list_bkup_dir()
{
   BkupDir=${1}
   if [ -d ${BkupDir} ]
   then
      log "Listing ${BkupDir}"
      ls -lisart ${BkupDir} | tee -a ${LOGFILE}
   fi
}


#
# Check for Hanging Backup
#
check_for_hanging_backup()
{
   Oracle_Sid=${1}
   Start_Date=${2}

   # Create Lock File for this backup to help determine if backup is hanging
   #
   LockFile=/tmp/backup.${Oracle_Sid}.${Start_Date}.lck
   touch ${LockFile}

   BkupTimeLimit=`get_dbmodes_value ${Oracle_Sid} BKUPTIMELIMIT 60`

   log "ORACLE_SID = ${Oracle_Sid} : BKUPTIMELIMIT = ${BkupTimeLimit}"

   # Start background process to sleep for $BKUPTIMELIMIT minutes and then verify
   # the backup for this SID has completed using the lock file created above
   # (if the lock file still exists after sleeping, then the backup must have hung)

   ${ADMIN_DIR}/check_backup_status.sh ${Oracle_Sid} ${BkupTimeLimit} ${LockFile} &
}


#
# Cleanup after the Backup Ends
#
end_backup()
{
   Oracle_Sid=${1}
   Start_Date=${2}

   # Remove Lock File created to determine if backup is hanging
   #
   LockFile=/tmp/backup.${Oracle_Sid}.${Start_Date}.lck
   rm -f ${LockFile}
}


#
# Backup the Online Redo Logs for Cold Backups
#
bkup_redo_logs()
{
   Oracle_Sid=${1}
   BkupDir=${2}
   log "Backing up online redo logs"
   RedoLogsDir=`get_dbmodes_value ${Oracle_Sid} REDOLOGSDIR N`
   RedoLogDirList=`echo ${RedoLogsDir} | sed "s/,/ /g"`
   for Field in ${RedoLogDirList}
   do
      for RedoLog in `ls ${Field}/*redo*`
      do
         RedoTarget=${BkupDir}/`basename ${RedoLog}`.`date +%Y%m%d%H%M`
         cp ${RedoLog} ${RedoTarget} | tee -a ${LOGFILE}
         compress ${RedoTarget}      | tee -a ${LOGFILE}
      done
   done
}


#
# Disable Previous Backups
#
disable_backups()
{
   log "Disabling previous backups"
   ${ORACLE_HOME}/bin/sqlplus -s '/ as sysdba' @${SQL_DIR}/disable_backups.sql | tee -a ${LOGFILE}
   rm -f /tmp/disable_backups.lst
   rm -f /tmp/disable_backups.sql
}

#
# Backup the Database to Tape
#
bkup_db_to_tape()
{
   BACKUP_POLICY=$1

   NB_ORACLE_SERVER=`grep SERVER /usr/openv/netbackup/bp.conf | grep -v _SERVER | cut -d "=" -f2 | tr -d " "`

   # BKRETENT is the number of days to keep backups.
   # If BKRETENT is not set in dbmodes.txt then set the default REDUNDANCY to 31.
   #
   BKRETENT=`get_dbmodes_value ${ORACLE_SID} TAPERETENT 31`

   log "ORACLE_SID = ${ORACLE_SID} : Backup Retention = ${BKRETENT} days"

   TAG_DATE_FORMAT=%Y_%m_%d_%H%M
   TAG_DATE=`date +${TAG_DATE_FORMAT}`

   log "ORACLE_SID = ${ORACLE_SID} : Copying the database backup sets to tape using Veritas NetBackkup"

   unset NLS_DATE_FORMAT
   # export NLS_DATE_FORMAT='DD-MON-YYYY HH24:MI:SS'

   ${ORACLE_HOME}/bin/rman target / catalog ${RMANDB_USERNAME}/${RMANDB_PASSWORD}@rmancatc.snt.bst.bls.com << EOF >> ${LOGFILE}

     CONFIGURE RETENTION POLICY TO NONE;
     CONFIGURE CONTROLFILE AUTOBACKUP OFF;
     CONFIGURE DEVICE TYPE SBT_TAPE BACKUP TYPE TO COMPRESSED BACKUPSET;
     CONFIGURE BACKUP OPTIMIZATION ON;

     run {
          allocate channel 't1' type 'SBT_TAPE' PARMS='ENV=(NB_ORA_SERV=${NB_ORACLE_SERVER},
          NB_ORA_CLIENT=$HOST,NB_ORA_POLICY=${BACKUP_POLICY},NB_ORA_SCHED=Default-Application-Backup)';

          allocate channel 't2' type 'SBT_TAPE' PARMS='ENV=(NB_ORA_SERV=${NB_ORACLE_SERVER},
          NB_ORA_CLIENT=$HOST,NB_ORA_POLICY=${BACKUP_POLICY},NB_ORA_SCHED=Default-Application-Backup)';

          allocate channel 't3' type 'SBT_TAPE' PARMS='ENV=(NB_ORA_SERV=${NB_ORACLE_SERVER},
          NB_ORA_CLIENT=$HOST,NB_ORA_POLICY=${BACKUP_POLICY},NB_ORA_SCHED=Default-Application-Backup)';

          allocate channel 't4' type 'SBT_TAPE' PARMS='ENV=(NB_ORA_SERV=${NB_ORACLE_SERVER},
          NB_ORA_CLIENT=$HOST,NB_ORA_POLICY=${BACKUP_POLICY},NB_ORA_SCHED=Default-Application-Backup)';

          backup backupset all;
          release channel t1;
          release channel t2;
          release channel t3;
          release channel t4;
     }

     allocate channel for maintenance device type 'SBT_TAPE'
     PARMS='ENV=(NB_ORA_SERV=${NB_ORACLE_SERVER},NB_ORA_CLIENT=$HOST,NB_ORA_POLICY=${BACKUP_POLICY},NB_ORA_SCHED=Default-Application-Backup)';
     DELETE NOPROMPT FORCE OBSOLETE RECOVERY WINDOW OF 31 DAYS;

     release channel ;

     # report obsolete ;
     # delete force noprompt obsolete device type 'SBT_TAPE' ;
     # crosscheck backup device type 'SBT_TAPE' ;
     # delete force noprompt expired backup device type 'SBT_TAPE' ;
     # list backup summary device type disk ;
     list backup summary device type 'SBT_TAPE' ;

     exit;

EOF
   Status=${?}
   return ${Status}
}


#
# Get Backup Directories
#
get_bkup_dirs()
{
   case `hostname` in

   nmsjs6d1|nmsjs6d2|nmsjs6d3|nmsjs6d4)
        BACKUPDIR=+JBBOS_RECOVERY
        BACKUPDIR1=+JBBOS_RECOVERY
        BACKUPDIR2=+JBBOS_RECOVERY
        BACKUPDIR3=+JBBOS_RECOVERY
        BACKUPDIR4=+JBBOS_RECOVERY
        ;;
    n5004002|n5004005|g0212021)
        BACKUPDIR=/orabackup1/dumpdata
        BACKUPDIR1=/orabackup1/dumpdata
        BACKUPDIR2=/orabackup2/dumpdata
        BACKUPDIR3=/orabackup3/dumpdata
        BACKUPDIR4=/orabackup4/dumpdata
        ;;
    sogbuat1)
        BACKUPDIR=/orabackup01/dumpdata
        BACKUPDIR1=/orabackup01/dumpdata
        BACKUPDIR2=/orabackup02/dumpdata
        BACKUPDIR3=/orabackup03/dumpdata
        BACKUPDIR4=/orabackup04/dumpdata
        ;;
    optiersb)
        BACKUPDIR=/pac/optier/bkup/orabkup01/dumpdata01
        BACKUPDIR1=/pac/optier/bkup/orabkup01/dumpdata01
        BACKUPDIR2=/pac/optier/bkup/orabkup01/dumpdata01
        BACKUPDIR3=/pac/optier/bkup/orabkup01/dumpdata01
        BACKUPDIR4=/pac/optier/bkup/orabkup01/dumpdata01
        ;;
    ads00620|ads00621|ads00644|ads00645|ads00616|ads00617|ads00648|ads00649|nmsf1n1|nmsf1n2|nmsf1n3|nmsf1n4|nmsf1n5|nmsf1n6|nmsf1n7|nmsf1n8|nmsp2n1|nmsp2n2|nmsp2n3|nm
sp2n4)
        BACKUPDIR=/orabackup01/dumpdata
        BACKUPDIR1=${BACKUPDIR}
        BACKUPDIR2=${BACKUPDIR}
        BACKUPDIR3=${BACKUPDIR}
        BACKUPDIR4=${BACKUPDIR}
        ;;
    ads00641|ads00642|lstst06|lstst11|nrsb_fs5|nrsb_fs6|ads00610|ads00609)
        BACKUPDIR=/pac/${ORACLE_SID}/bkup/orabkup01/dumpdata01
        BACKUPDIR1=/pac/${ORACLE_SID}/bkup/orabkup01/dumpdata01
        BACKUPDIR2=/pac/${ORACLE_SID}/bkup/orabkup01/dumpdata02
        BACKUPDIR3=/pac/${ORACLE_SID}/bkup/orabkup01/dumpdata03
        BACKUPDIR4=/pac/${ORACLE_SID}/bkup/orabkup01/dumpdata04
        ;;
    bcsldt10)
        BACKUPDIR=/pac/${ORACLE_SID}/bkup/orabkup01/dumpdata01
        BACKUPDIR1=${BACKUPDIR}
        BACKUPDIR2=${BACKUPDIR}
        BACKUPDIR3=${BACKUPDIR}
        BACKUPDIR4=${BACKUPDIR}
        ;;
    *)
        # BACKUPDIR=/pac/cam/bkup/orabkup01/dumpdata01
        BACKUPDIR=`get_Backup_Dir ${ORACLE_SID}`
        BACKUPDIR1=${BACKUPDIR}
        BACKUPDIR2=${BACKUPDIR}
        BACKUPDIR3=${BACKUPDIR}
        BACKUPDIR4=${BACKUPDIR}
        ;;
   esac

}

#
# Backup the Database
#
bkup_db_to_disk()
{
   # BKRETENT is the number of days to keep backups.
   # If BKRETENT is not set in dbmodes.txt then set the default REDUNDANCY to 5.
   #
   BKRETENT=`get_dbmodes_value ${ORACLE_SID} BKRETENT 5`

   log "ORACLE_SID = ${ORACLE_SID} : Backup Retention = ${BKRETENT} days"

   SPFILEFLAG=`get_spfile_flag`

   log "ORACLE_SID = ${ORACLE_SID} : spfileflag=${SPFILEFLAG}"

   TAG_DATE_FORMAT=%Y_%m_%d_%H%M
   TAG_DATE=`date +${TAG_DATE_FORMAT}`

   log "ORACLE_SID = ${ORACLE_SID} : Backing up the database with RMAN"

   export NLS_DATE_FORMAT='DD-MON-YYYY HH24:MI:SS'

   ${ORACLE_HOME}/bin/rman target / catalog ${RMANDB_USERNAME}/${RMANDB_PASSWORD}@rmancatc.snt.bst.bls.com << EOF >> ${LOGFILE}

     # Note: You cannot assign a tag to a backup of the current control file.
     #
     CONFIGURE CONTROLFILE AUTOBACKUP FORMAT FOR DEVICE TYPE DISK TO '${BACKUPDIR}/cf_%F';
     CONFIGURE CONTROLFILE AUTOBACKUP ON;
     CONFIGURE RETENTION POLICY TO RECOVERY WINDOW OF ${BKRETENT} DAYS;
     CONFIGURE DEVICE TYPE DISK PARALLELISM 4 BACKUP TYPE TO COMPRESSED BACKUPSET;
     CONFIGURE BACKUP OPTIMIZATION OFF;
     ALLOCATE CHANNEL FOR MAINTENANCE DEVICE TYPE DISK;
     CONFIGURE CHANNEL DEVICE TYPE 'SBT_TAPE' CLEAR;

     run {

       crosscheck backup device type disk ;
       crosscheck archivelog all;
        crosscheck backup of archivelog all spfile;
        delete force noprompt obsolete device type disk;
        delete force noprompt expired backup device type disk;
        delete force noprompt expired archivelog all device type disk;
        delete force noprompt backupset device type disk completed before 'sysdate-${BKRETENT}';

       # CR Copy ALTER between blocks
       # Oracle auto does a LOG SWITCH as part of BACKUP block
       #
      sql 'alter system switch logfile';

      sql 'alter system archive log current';

       allocate channel c1 device type disk format='${BACKUPDIR1}/${TAG_DATE}_dbf_s%s_p%p';
       allocate channel c2 device type disk format='${BACKUPDIR2}/${TAG_DATE}_dbf_s%s_p%p';
       allocate channel c3 device type disk format='${BACKUPDIR3}/${TAG_DATE}_dbf_s%s_p%p';
       allocate channel c4 device type disk format='${BACKUPDIR4}/${TAG_DATE}_dbf_s%s_p%p';

       # allocate channel c1 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';
       # allocate channel c2 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';
       # allocate channel c3 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';
       # allocate channel c4 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';

       backup
          incremental level 0
          filesperset 4
          # format '${BACKUPDIR}/${TAG_DATE}_dbf_s%s_p%p'
          database
          tag = '${TAG_DATE}_bkup_db'
          include current controlfile for standby;

       release channel c1;
       release channel c2;
       release channel c3;
       release channel c4;

     }
     exit;

     # ${SPFILEFLAG} ;
     # exit;

     # list backup summary device type disk ;
EOF

   Status=${?}
   return ${Status}
}

#
# Incremental Level 1 Backup of  the Database (Below Logic is appended by Pankaj)
#
incremental_bkup_db_to_disk()
{
   # BKRETENT is the number of days to keep backups.
   # If BKRETENT is not set in dbmodes.txt then set the default REDUNDANCY to 5.
   #
   BKRETENT=`get_dbmodes_value ${ORACLE_SID} BKRETENT 5`

   log "ORACLE_SID = ${ORACLE_SID} : Backup Retention = ${BKRETENT} days"

   SPFILEFLAG=`get_spfile_flag`

   log "ORACLE_SID = ${ORACLE_SID} : spfileflag=${SPFILEFLAG}"

   TAG_DATE_FORMAT=%Y_%m_%d_%H%M
   TAG_DATE=`date +${TAG_DATE_FORMAT}`

   log "ORACLE_SID = ${ORACLE_SID} : Backing up the database with RMAN"

   export NLS_DATE_FORMAT='DD-MON-YYYY HH24:MI:SS'

   ${ORACLE_HOME}/bin/rman target / catalog ${RMANDB_USERNAME}/${RMANDB_PASSWORD}@rmancatc.snt.bst.bls.com << EOF >> ${LOGFILE}

     # Note: You cannot assign a tag to a backup of the current control file.
     #
     CONFIGURE CONTROLFILE AUTOBACKUP FORMAT FOR DEVICE TYPE DISK TO '${BACKUPDIR}/cf_%F';
     CONFIGURE CONTROLFILE AUTOBACKUP ON;
     CONFIGURE RETENTION POLICY TO RECOVERY WINDOW OF ${BKRETENT} DAYS;
     CONFIGURE DEVICE TYPE DISK PARALLELISM 4 BACKUP TYPE TO COMPRESSED BACKUPSET;
     CONFIGURE BACKUP OPTIMIZATION OFF;
     ALLOCATE CHANNEL FOR MAINTENANCE DEVICE TYPE DISK;
     CONFIGURE CHANNEL DEVICE TYPE 'SBT_TAPE' CLEAR;

     run {

       crosscheck backup device type disk ;
       crosscheck archivelog all;
       crosscheck backup of archivelog all spfile;
       delete force noprompt obsolete device type disk;
       delete force noprompt expired backup device type disk;
       delete force noprompt expired archivelog all device type disk;

       # CR Copy ALTER between blocks
       # Oracle auto does a LOG SWITCH as part of BACKUP block
       #
      sql 'alter system switch logfile';

      sql 'alter system archive log current';

       allocate channel c1 device type disk format='${BACKUPDIR1}/${TAG_DATE}_dbf_s%s_p%p';
       allocate channel c2 device type disk format='${BACKUPDIR2}/${TAG_DATE}_dbf_s%s_p%p';
       allocate channel c3 device type disk format='${BACKUPDIR3}/${TAG_DATE}_dbf_s%s_p%p';
       allocate channel c4 device type disk format='${BACKUPDIR4}/${TAG_DATE}_dbf_s%s_p%p';

       # allocate channel c1 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';
       # allocate channel c2 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';
       # allocate channel c3 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';
       # allocate channel c4 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_dbf_s%s_p%p';

       backup
          incremental level 1
          filesperset 4
          # format '${BACKUPDIR}/${TAG_DATE}_dbf_s%s_p%p'
          database
          tag = '${TAG_DATE}_bkup_db'
          include current controlfile for standby;

       release channel c1;
       release channel c2;
       release channel c3;
       release channel c4;

     }
     exit;

     # ${SPFILEFLAG} ;
     # exit;

     # list backup summary device type disk ;
EOF

   Status=${?}
   return ${Status}
}

#
# Backup the Archived Logs
#
bkup_arch_to_disk()
{
   # BKRETENT is the number of days to keep backups.
   # If BKRETENT is not set in dbmodes.txt then set the default REDUNDANCY to 5.
   #
   BKRETENT=`get_dbmodes_value ${ORACLE_SID} BKRETENT 5`

   log "ORACLE_SID = ${ORACLE_SID} : Backup Retention = ${BKRETENT} days"

   TAG_DATE_FORMAT=%Y_%m_%d_%H%M
   TAG_DATE=`date +${TAG_DATE_FORMAT}`

   log "ORACLE_SID = ${ORACLE_SID} : Backing up the archived logs with RMAN"

   export NLS_DATE_FORMAT='DD-MON-YYYY HH24:MI:SS'

   ${ORACLE_HOME}/bin/rman target / catalog ${RMANDB_USERNAME}/${RMANDB_PASSWORD}@rmancatc.snt.bst.bls.com << EOF >> ${LOGFILE}

     CONFIGURE CONTROLFILE AUTOBACKUP FORMAT FOR DEVICE TYPE DISK TO '${BACKUPDIR}/cf_%F';
     CONFIGURE CONTROLFILE AUTOBACKUP ON;
     CONFIGURE RETENTION POLICY TO RECOVERY WINDOW OF ${BKRETENT} DAYS;
     CONFIGURE DEVICE TYPE DISK PARALLELISM 4 BACKUP TYPE TO COMPRESSED BACKUPSET;
     CONFIGURE BACKUP OPTIMIZATION OFF;
     ALLOCATE CHANNEL FOR MAINTENANCE DEVICE TYPE DISK;
     CONFIGURE CHANNEL DEVICE TYPE 'SBT_TAPE' CLEAR;

     run {

       crosscheck backup device type disk ;
       crosscheck archivelog all device type disk;
       # crosscheck backup of archivelog all spfile;
        delete force noprompt obsolete device type disk;
        delete force noprompt expired backup device type disk;
        delete force noprompt expired archivelog all device type disk;
        delete force noprompt backupset device type disk completed before 'sysdate-${BKRETENT}';

       # CR Copy ALTER between blocks
       # ?? Oracle auto does a LOG SWITCH as part of BACKUP block
       # Oracle does an ALTER SYSTEM ARCHIVE LOG CURRENT as part of the BACKUP ARCHIVELOG block
       #
       sql 'alter system switch logfile';

       sql 'alter system archive log current';

       allocate channel c1 device type disk format='${BACKUPDIR1}/${TAG_DATE}_arc_s%s_p%p';
       allocate channel c2 device type disk format='${BACKUPDIR2}/${TAG_DATE}_arc_s%s_p%p';
       allocate channel c3 device type disk format='${BACKUPDIR3}/${TAG_DATE}_arc_s%s_p%p';
       allocate channel c4 device type disk format='${BACKUPDIR4}/${TAG_DATE}_arc_s%s_p%p';

       # allocate channel c1 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_arc_s%s_p%p';
       # allocate channel c2 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_arc_s%s_p%p';
       # allocate channel c3 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_arc_s%s_p%p';
       # allocate channel c4 device type disk format='/pac/cam/bkup/orabkup01/dumpdata01/${TAG_DATE}_arc_s%s_p%p';

       backup
          incremental level 0
          filesperset 4
          # format '${BACKUPDIR}/${TAG_DATE}_arc_s%s_p%p'
          archivelog all delete input
          # archivelog all
          tag = '${TAG_DATE}_bkup_arch';

       release channel c1;
       release channel c2;
       release channel c3;
       release channel c4;

       # sql 'alter database backup controlfile to trace';

       crosscheck archivelog all device type disk;
       delete force noprompt obsolete device type disk;
     }

     list backup summary device type disk ;

     exit;

EOF
   Status=${?}
   return ${Status}
}


#
# Get the Overall Backup Status
#
get_bkup_status()
{
    Prev_Status=${1}
    Status_Code=${2}
    New_Status=${Prev_Status}
    if [ ${Prev_Status} = INPROGRESS ]
    then
       if [ ${Status_Code} -eq 0 ]
       then
          New_Status=SUCCESSFUL
       else
          New_Status=UNSUCCESSFUL
       fi
    else
       if [ ${Prev_Status} != UNSUCCESSFUL -a ${Status_Code} != 0 ]
       then
          New_Status=WARNING
       fi
    fi
    echo ${New_Status}
}


###########################################
#
#                   Main
#
###########################################

ARCHIVEONLY=N
BKUPTOTAPE=N
INCREMENTALONLY=N
RMANDB_USERNAME=rman102
RMANDB_PASSWORD=cat

USAGE="USAGE - `basename ${0}` [-a] archiveonly [-s SID] [-i] incremental [-t] tape backup [-d] data domain backup [-h] help"


while getopts :aitdhs: opt;
do
   case "${opt}" in
      a) ARCHIVEONLY=Y
          ;;
      i) INCREMENTALONLY=Y
          ;;
      t) BKUPTOTAPE=T
         T_MODE=1
          ;;
      d) BKUPTOTAPE=D
         D_MODE=1
          ;;
      h) echo ""
         echo "   $USAGE"
         echo ""
         echo "   -a archive log backup only"
         echo "   -s if you have multiple database on one server"
         echo "       and only want to backup one - REQUIRES an argument."
         echo "   -i incrmental backup only"
         echo "   -t tape backup"
         echo "   -d data domain backup"
         echo "   -h display help"
         echo ""
         exit
          ;;
      s) ORACLE_SID_LIST=$OPTARG
         CMD_LINE_SID=Y
          ;;
      *) echo ""
         echo "   $USAGE"
         echo ""
         echo "   -a archive log backup only"
         echo "   -s if you have multiple database on one server"
         echo "       and only want to backup one - REQUIRES an argument."
         echo "   -i incrmental backup only"
         echo "   -t tape backup"
         echo "   -d data domain backup"
         echo "   -h display help"
         echo ""
         exit
          ;;
   esac
done


if [ -x /usr/ucb/whoami ]
then
   WHOAMI=/usr/ucb/whoami
else
   WHOAMI=whoami
fi

#if [ `${WHOAMI}` = oracle ]
#then
   . ~oracle/oracle_env
#else
  if [ -x ~/.local ]
  then
     . ~/.local
  else
     echo "###################################"
     echo "# ERROR - ~/.local does not exist #"
     echo "###################################"
     exit
  fi
#fi

#
# Define the LOGFILE
#
format_backup_logfile_name

if [ T_MODE -eq 1 ] && [ D_MODE -eq 1 ]
then
  echo "#######################################################" >> ${LOGFILE}
  echo "" >> ${LOGFILE}
  echo "Tape Mode and Data Domain Mode are exclusive - Pick One" >> ${LOGFILE}
  echo "" >> ${LOGFILE}
  echo "$USAGE" >> ${LOGFILE}
  echo "" >> ${LOGFILE}
  echo "#######################################################" >> ${LOGFILE}
  exit -1
fi

# Load Common Functions
#
. ${ADMIN_DIR}/common_functions.sh

# Set the Date Format
#
DATE_FORMAT=%Y%m%d%H%M


# Set the Oracle Environment
#

# Get the list of ORACLE_SIDs
#

if [ -z ${CMD_LINE_SID} ]
then
   ORACLE_SID_LIST=`list_oracle_sids`
fi

log "HOST = ${HOST}"

for ORACLE_SID in ${ORACLE_SID_LIST}
do

   # Determine whether SID should be Processed
   #
   #PROCESS_FLAG=`get_process_flag ${ORACLE_SID}`

   if [ ${ORACLE_SID} = 'rman' ]
   then
      PROCESS_FLAG = N
   fi
   log "ORACLE_SID = ${ORACLE_SID} : Process Flag = ${PROCESS_FLAG}"

   if [ ${PROCESS_FLAG} = N ]
   then

      log "Skipping backup of ${ORACLE_SID}"

   else

      log "ORACLE_SID = ${ORACLE_SID} : Processing ..."

      if [ -x /usr/ucb/whoami ]
      then
         WHOAMI=/usr/ucb/whoami
      else
         WHOAMI=whoami
      fi

#      if [ `${WHOAMI}` = oracle ]
#      then
#         . ~oracle/oracle_env ${ORACLE_SID}
#      else
#         . ~/oracle/local/bin/oraprof ${ORACLE_SID}
#      fi

. ~oracle/oracle_env ${ORACLE_SID}
. ~/oracle/local/bin/oraprof ${ORACLE_SID}

      START_DATE=`date +'%Y%m%d%H%M%S'`

      BKUPDISABLE=`get_dbmodes_value ${ORACLE_SID} BKUPDISABLE N`

      if [ ${BKUPDISABLE} = Y ]
      then

         log "Backups disabled for Host = ${HOST} : ORACLE_SID = ${ORACLE_SID}"

         REPORT_STATUS='DISABLED'
         ${ADMIN_DIR}/insert_backup_status.sh ${ORACLE_SID} ${REPORT_STATUS} ${START_DATE} ${START_DATE} | tee -a ${LOGFILE}

      else

         DBMODE=`get_dbmodes_value ${ORACLE_SID} DBMODE PRIMARY`

         log "ORACLE_SID = ${ORACLE_SID} : DBMODE = ${DBMODE}"

         if [ ${DBMODE} = STANDBY ]
         then

            log "SID=${ORACLE_SID} : Skipping backup for Standby Database"

            REPORT_STATUS='STANDBY'
            ${ADMIN_DIR}/insert_backup_status.sh ${ORACLE_SID} ${REPORT_STATUS} ${START_DATE} ${START_DATE} | tee -a ${LOGFILE}

         else

            ORACLE_HOME=`get_Oracle_Home ${ORACLE_SID}`

            log "ORACLE_SID = ${ORACLE_SID} : ORACLE_HOME = ${ORACLE_HOME}"

            # BACKUPDIR=`get_Backup_Dir ${ORACLE_SID}`

            get_bkup_dirs

            log "ORACLE_SID = ${ORACLE_SID} : Backup Directory   = ${BACKUPDIR}"
            log "ORACLE_SID = ${ORACLE_SID} : Backup Directory 1 = ${BACKUPDIR1}"
            log "ORACLE_SID = ${ORACLE_SID} : Backup Directory 2 = ${BACKUPDIR2}"
            log "ORACLE_SID = ${ORACLE_SID} : Backup Directory 3 = ${BACKUPDIR3}"
            log "ORACLE_SID = ${ORACLE_SID} : Backup Directory 4 = ${BACKUPDIR4}"


            BKUPTYP=`get_dbmodes_value ${ORACLE_SID} BKUPTYP ONLINE`

            log "ORACLE_SID = ${ORACLE_SID} : BKUPTYP = ${BKUPTYP}"

            REPORT_STATUS='INPROGRESS'
            ${ADMIN_DIR}/insert_backup_status.sh ${ORACLE_SID} ${REPORT_STATUS} ${START_DATE} | tee -a ${LOGFILE}


            # Start background process to sleep for $BKUPTIMELIMIT minutes and then verify
            # the backup for this SID has completed using the lock file created above
            # (if the lock file still exists after sleeping, then the backup must have hung)
            #
            # check_for_hanging_backup ${ORACLE_SID} ${START_DATE}

            STATUS=0

            # List Contents of Backup Directory before Backup
            #
            list_bkup_dir ${BACKUPDIR}

            log "Starting ${BKUPTYPE} Backup"

            # Initialize Report Status
            #
            # REPORT_STATUS=UNKNOWN

              # Incremental Backup Of The Database
              #
            if [ ${INCREMENTALONLY} = Y ]
            then

              incremental_bkup_db_to_disk
              # echo "Doing Incremental Backup" $INCREMENTALONLY $ARCHIVEONLY

              STATUS=${?}

              # Log the Exit Code
              #
              log "ORACLE_SID = ${ORACLE_SID} : Incremental Backup of database completed with return code of ${STATUS}"

              REPORT_STATUS=`get_bkup_status ${REPORT_STATUS} ${STATUS}`

              log "ORACLE_SID = ${ORACLE_SID} : Report Status = ${REPORT_STATUS}"

              ARCHIVEONLY=Y

            fi

            if [ ${ARCHIVEONLY} != Y ]
            then

              # Full Backup Of The Database
              #
              bkup_db_to_disk

              #echo "Doing Full Backup" $INCREMENTALONLY $ARCHIVEONLY

              STATUS=${?}

              # Log the Exit Code
              #
              log "ORACLE_SID = ${ORACLE_SID} : Backup of database completed with return code of ${STATUS}"

              REPORT_STATUS=`get_bkup_status ${REPORT_STATUS} ${STATUS}`

              log "ORACLE_SID = ${ORACLE_SID} : Report Status = ${REPORT_STATUS}"

            fi

            # Backup the Archived Logs
            #
            bkup_arch_to_disk

            STATUS=${?}

            # Log the Exit Code
            #
            log "ORACLE_SID = ${ORACLE_SID} : Backup of archived logs completed with return code of ${STATUS}"

            REPORT_STATUS=`get_bkup_status ${REPORT_STATUS} ${STATUS}`

            log "ORACLE_SID = ${ORACLE_SID} : Report Status = ${REPORT_STATUS}"

            if [ ${BKUPTOTAPE} = T ]
            then

              log "ORACLE_SID = ${ORACLE_SID} : Backup sets will be transferred to tape"

              # Transfer the Database Backup Sets to Tape
              #
              bkup_db_to_tape Oracle-RMAN

              STATUS=${?}

              # Log the Exit Code
              #
              log "ORACLE_SID = ${ORACLE_SID} : Transfer of backup sets to tape completed with return code of ${STATUS}"

              REPORT_STATUS=`get_bkup_status ${REPORT_STATUS} ${STATUS}`

              log "ORACLE_SID = ${ORACLE_SID} : Report Status = ${REPORT_STATUS}"

            fi

            if [ ${BKUPTOTAPE} = D ]
            then

              log "ORACLE_SID = ${ORACLE_SID} : Backup sets will be transferred to tape"

              # Transfer the Database Backup Sets to Data Domain
              #
              bkup_db_to_tape Oracle-RMAN-DD

              STATUS=${?}

              # Log the Exit Code
              #
              log "ORACLE_SID = ${ORACLE_SID} : Transfer of backup sets to Data Domain completed with return code of ${STATUS}"

              REPORT_STATUS=`get_bkup_status ${REPORT_STATUS} ${STATUS}`

              log "ORACLE_SID = ${ORACLE_SID} : Report Status = ${REPORT_STATUS}"

            fi


            # Copy init.ora File to Alternate Location
            # ??? Do we need or want to do this ???
            # ??? Umesh and Romayne say yes ???
            #
            # if [ -s ${ORACLE_HOME}/dbs/init${ORACLE_SID}.ora ]
            # then
            #     cp ${ORACLE_HOME}/dbs/init${ORACLE_SID}.ora ${BACKUPDIR}/init${ORACLE_SID}.ora.`date +%d-%m-%Y.%H:%M:%S`
            #     cp ${ORACLE_HOME}/dbs/init${ORACLE_SID}.ora ${BACKUPDIR}/${TAG_DATE}_init${ORACLE_SID}
            # fi

            # Copy Password File to Alternate Location
            # ??? Do we need or want to do this ???
            # if [ -s ${ORACLE_HOME}/dbs/orapw${ORACLE_SID} ]
            # then
            #     cp ${ORACLE_HOME}/dbs/orapw${ORACLE_SID} ${BACKUPDIR}/orapw${ORACLE_SID}.`date +%d-%m-%Y.%H:%M:%S`
            #     cp ${ORACLE_HOME}/dbs/orapw${ORACLE_SID} ${BACKUPDIR}/${TAG_DATE}_orapw${ORACLE_SID}
            # fi

            # Set the Status for Report
            if [ ${REPORT_STATUS} = UNSUCCESSFUL ]
            then
               #BKUPFAILALERT=`get_dbmodes_value ${ORACLE_SID} BKUPFAILALERT N`
               BKUPFAILALERT=`get_dbmodes_value ${ORACLE_SID} BKUPFAILALERT`

               log "ORACLE_SID = ${ORACLE_SID} : BKUPFAILALERT = ${BKUPFAILALERT}"

               if [ ${BKUPFAILALERT} = Y ]
               then
                   ${ADMIN_DIR}/send_mail.sh critical "Backup Failure" "${HOST}:${ORACLE_SID}:Backup Failed"
               fi
            fi

            # Log the Backup Status
            #
            log "RMAN: The Full Online Backup of ${ORACLE_SID} was ${REPORT_STATUS}"

            # List Contents of Backup Directory after Backup
            #
            list_bkup_dir ${BACKUPDIR}

         fi

         # Cleanup after the Backup is done
         #
         end_backup ${ORACLE_SID} ${START_DATE}

         # 06/08/2006 SRF - replaced call to report_backup_status.sh with call to update_backup_status.sh
         #                  which updates the record written earlier in this script (key is HOST, SID
         #                  and START DATE) - adding the END DATE and changing the STATUS
         ${ADMIN_DIR}/update_backup_status.sh ${ORACLE_SID} ${REPORT_STATUS} ${START_DATE} | tee -a ${LOGFILE}

         # 07/02/2008     -  The script update_rman_transition_status.sh is called to update the backup type used
         #                   by the database,this information is used to track the transition of backups
         #                   to RMAN.
         ${ADMIN_DIR}/update_rman_transition_status.sh ${HOST} ${ORACLE_SID} RMAN | tee -a ${LOGFILE}

      fi

   fi

done

#
# Exit the Script with the Exit Status
#
exit ${STATUS}

FIG project queries

##### Service add & LOad Baclancing on Add Service ####### srvctl add service -s wcccdmt.farmersinsurance.com -r wcccdmtx1,wcccdmtx2,wcc...