# Copyright (c) 2011, 2018, Oracle and/or its affiliates. All rights reserved. # # NAME # crsupgrade.pm # # DESCRIPTION # This module contains upgrade related functions for CRS/SIHA # # NOTES # # # MODIFIED (MM/DD/YY) # apfwkr 05/31/18 - Backport # shullur_ci_backport_28039471_18.0.0.0.0ocwru from # st_has_18.0 # shullur 04/16/18 - Backport shullur_bug-27740854 from main # apfwkr 03/07/18 - Backport sbezawad_bug27285557 from main # muhe 02/23/18 - Backport muhe_bug-27212837 from main # xyuan 11/29/17 - Explicitly stop the rhpserver before stopping the # older stack as a workaround for bug#26979409 # shullur 03/30/18 - For fixing bug 27740854, export CHA models. # sbezawad 02/02/18 - Fix bug 27285557. Set clusterware to rolling # migration mode for far ASM # muhe 01/08/18 - Fix bug 27212837 # xyuan 11/06/17 - Fix bug 27046294 # muhe 11/02/17 - Fix bug 27060822 # muhe 10/25/17 - Fix bug 27014998 # xyuan 10/19/17 - Fix bug 26994867 # muhe 10/11/17 - Fix bug 26939269 # shiyer 10/03/17 - #26895782:cleanup sandbox processes # lcarvall 09/27/17 - Fix bug 26791882 haip is disabled after an addnode # operation in the exadata environment # jolascua 09/07/17 - Bug 26654615: ASM cred domain not found when # upgrading from 12.1 to 12.2 # dpham 08/28/17 - Fix bug 26679664 # muhe 08/24/17 - Fix bug 26664486 # shxi 08/24/17 - Fix bug 26672778 # luoli 08/01/17 - Fix bug 26554987 # muhe 07/26/17 - Fix bug 26449535 # rdasari 07/21/17 - handle rerun for cluster root cert # muhe 07/13/17 - Fix bug 26375531 # rdasari 07/10/17 - remove get_HAIP_ENABLED_fromOld # xyuan 07/06/17 - Fix rti 20431246 # shiyer 06/30/17 - #26171072:dry run checkpoint,#26263895 # muhe 06/28/17 - Fix bug 26356303 # luoli 06/22/17 - Remove CDATA_ALL_DISKGROUPS # xyuan 06/19/17 - Flipped UpgradeAFD and UpgradeAFD steps for SIHA # upgrade # luoli 06/16/17 - Fix bug 26171113 # sbezawad 06/14/17 - Bug 26245551: Add CRS home to updateHistory # xyuan 06/08/17 - Fix bug 26132404 # sbezawad 06/07/17 - Bug 26172414: During dry run, export OCR before # setting OCR dev env # luoli 06/01/17 - Fix bug 26147532 # xyuan 05/24/17 - Fix bug 26126901 # muhe 05/15/17 - Remove duplicate functions # shiyer 05/15/17 - lrg #20231778:call wait_for_sandbox_crsd_exit() # muhe 05/08/17 - Fix bug 25996923 # siyarlag 05/02/17 - Bug/20019427: use new root script framework for AFD # xyuan 04/19/17 - Fix bug 24514803 # shiyer 04/19/17 - #25908189:skip TFA for dry run # luoli 04/11/17 - Fix bug 25792868 # sbezawad 04/10/17 - Bug 25859187: Export sandbox location # luoli 04/09/17 - Fix bug 25759090 # xyuan 04/05/17 - Fix bug 25791441 # shiyer 03/28/17 - #25776304 # xyuan 03/24/17 - Fix bug 25775227 # muhe 03/17/17 - Fix bug 25514391 # muhe 03/07/17 - Support SIHA upgrade # sbezawad 03/05/17 - Bug 22829113: Update history # luoli 02/27/17 - Fix merge conflicts caused by bug 25448462 # madoming 02/09/17 - Fix bug 25518447 # shiyer 02/08/17 - Dry run bugs # muhe 02/07/17 - Fix bug 25479359 # muhe 02/07/17 - Fix bug 25448462 # shiyer 01/14/17 - CRS Upgrade pre-check # mperezh 01/02/17 - Move ACFS first node enables to oraacfs.pm # lluis 12/12/16 - Fix bug 23093853 # muhe 12/12/16 - Support GIMR upgrade from 12.2 # luoli 10/11/16 - Fix bug 24665035 # dpham 09/26/16 - remove crs_* binaries (22992267) # muhe 09/23/16 - Fix bug 24673586 # muhe 09/22/16 - Fix bug 24706306 # bbeelamk 09/13/16 - Fix bug 24602312 # xyuan 09/13/16 - Fix bug 24588125 # luoli 08/12/16 - Fix bug 24452059 # luoli 07/31/16 - XbranchMerge luoli_bug-24311045 from # st_has_12.2.0.1.0 # luoli 07/28/16 - Fix bug 24311045 # muhe 07/19/16 - Fix bug 16976127 # muhe 07/14/16 - Fix bug 24287160 # luoli 07/10/16 - Fix bug 23762745 # luoli 07/04/16 - Add ckpt for SIHA upgrade # xyuan 06/28/16 - Fix bug 23707444 # luoli 06/22/16 - Fix bug 23623597 # muhe 06/14/16 - Fix bug 23192386 # xyuan 06/14/16 - Fix bug 23581473 # muhe 05/23/16 - Add check point for upgrade prechecks # xyuan 05/09/16 - Fix bug 22382998 # dpham 04/27/16 - add CRS_HOME parameter # luoli 04/26/16 - Fix bug 23184405 # luoli 04/13/16 - Fix a typo # bbeelamk 04/13/16 - Fix bug 23095140 # xyuan 04/07/16 - Fix bug 23016188 # yilhu 04/01/16 - Modify subroutine postupgrade_actions() # muhe 03/29/16 - Fix bug 23004889 # xyuan 03/24/16 - Fix bug 23000539 # muhe 03/16/16 - Fix bug 22936276 # luoli 03/15/16 - Fix bug 22910434 # luoli 03/15/16 - Fix bug 22894620 # bbeelamk 03/10/16 - Fix bug 22856205 # bbeelamk 03/06/16 - Fix srvctl error # dpham 02/16/16 - include crsxag.pm module # luoli 03/01/16 - Modify subroutine setASMNetworks() # mperezh 02/05/16 - Fix 22832595 - Don't use RESOURCE_LIST in the # last node. # mperezh 02/05/16 - Enable asm proxy in the last node before # upgrade_config() # luoli 02/02/16 - Fix lrg 18660895 # xyuan 01/27/16 - Fix bug 22578271 # jgrout 01/25/16 - Clean up bad whitespace # jgrout 01/25/16 - Fix bug 21419874 # xyuan 01/25/16 - Fix bug 22558143 # luoli 01/07/16 - Fix bug 22509821 # bbeelamk 12/20/15 - Fix bug 22124821 # rtamezd 12/16/15 - RHP repos upgrade changes # luoli 12/15/15 - Fix bug 21921412 # xyuan 12/03/15 - Fix bug 22283181 # madoming 11/27/15 - Fix bug 22273934 # luoli 11/22/15 - Fix bug 22185539 # xyuan 11/18/15 - Fix bug 21134068 # sbezawad 11/17/15 - Bug 21919798: Add cluster properties to clscfg # muhe 11/15/15 - Fix bug 22161744 # muhe 11/10/15 - Fix bug 22148653, 22162692 # xyuan 11/04/15 - Fix bug 22154174 # muhe 10/29/15 - Fix bug 22116434 # luoli 10/26/15 - Fix bug 22085137 # jmarcias 10/23/15 - Fix bug 22082682 # muhe 10/21/15 - Fix bug 22062104 # xyuan 10/18/15 - Fix bug 22016711 # luoli 10/13/15 - Branch preupgrade_actions for CRS and SIHA # bbeelamk 10/09/15 - Fix bug 21689973 # gmaldona 10/06/15 - Do not start OC4J # bbeelamk 09/30/15 - Fix bug 20679385 # muhe 09/22/15 - Fix bug 21863537 # bbeelamk 09/21/15 - Fix lrg 18483454 # bbeelamk 09/15/15 - Changing srvctl method # bbeelamk 09/10/15 - Fix bug 21761307 # xyuan 09/04/15 - Fix bug 21470281 # luoli 08/16/15 - Fix bug 20861530 # jmarcias 08/12/15 - Fix bug 21618216 # jmunozn 08/03/15 - Fix bug 21387018: stop/disable oc4j from old CRS # home # muhe 07/30/15 - Fix bug 21528944 # bbeelamk 07/30/15 - Fix bug 21489877 # luoli 07/28/15 - Add -y -z -h to CLSCFG command # luoli 07/20/15 - Fix bug 21470746 # bbeelamk 07/20/15 - Fix lrg 17723321 # muhe 07/08/15 - Fix bug 21047431 # bbeelamk 07/08/15 - Fix bug 21269876 # luoli 07/01/15 - Fix bug 21324323 # jmarcias 06/25/15 - Fix bug 21137634 # shullur 06/24/15 - For CHM migration modules to new rootscript changes # luoli 06/15/15 - Change print_error to print_info when printing # success information # xyuan 06/15/15 - Fix bug 21253660 # bbeelamk 06/11/15 - Fix lrg 16935885 # xyuan 06/10/15 - Fix bug 21205876 # muhe 06/10/15 - Fix bug 21141141 # luoli 06/07/15 - Fix bug 21213444 # muhe 05/31/15 - Fix bug 21064827 # agorla 05/28/15 - moving CVU to orajagent # xyuan 05/22/15 - Fix bug 21126418 # ksviswan 05/13/15 - use crscpcfg for copy action during OOP. # muhe 05/13/15 - Use print_info() for message 343 # bbeelamk 05/22/15 - Change ASM instance to flex on first node to upgrade # luoli 05/12/15 - Fix bug 21073362 # bbeelamk 05/08/15 - Fix bug 20950908 # xyuan 05/04/15 - Fixed one issue: copying GPnP stuff to newer # home twice during upgrade # xyuan 04/30/15 - Fix bug 20980356 # jinjche 04/29/15 - Back out change for bug 18006526 # xyuan 04/28/15 - Fix bug 20961590 # muhe 04/21/15 - Fix bug 20879013 # muhe 04/20/15 - Fix bug 20916630 # jinjche 04/20/15 - Fix an existing bug that failed to copy GPNP # profiles and wallets because it uses non-existent # directories # muhe 04/14/15 - Add pre checks for upgrade # xyuan 04/09/15 - Clean-ups # luoli 04/07/15 - Fix bug 20811547 # jmarcias 04/06/15 - Fix bug 20818194 # bbeelamk 04/01/15 - Fix bug 20567041 # gnagiah 03/30/15 - delete diagsnap resource in 12.2 and above # muhe 03/23/15 - Fix bug 20725601 # madoming 03/20/15 - Changes for new framework # emarron 03/20/15 - Use crska.pm # muhe 03/20/15 - Fix bug 20710572 # xyuan 03/19/15 - Changes for ugrading std ASM # bbeelamk 03/19/15 - Fix bug 20657928 # nkorehis 03/16/15 - bug-20136892: perform_CHM_upgrade calls s_setperms # bbeelamk 03/05/15 - Fix bug 20527524 # sbezawad 03/04/15 - Bug 20638245 # sbezawad 03/02/15 - Bug 20620752: Move setup_env after comp OCR init # xyuan 02/27/15 - Fix bug 20614945 # jmarcias 02/27/15 - Fix bug 20614165 # jmarcias 02/18/15 - Fix bug 20518463 # muhe 02/15/15 - Fix bug 20542987 # jmarcias 02/10/15 - Fix bug 20441873 # luoli 02/09/15 - Fix bug 19232406 # jmarcias 01/29/15 - Fix bug 20425262 # xyuan 01/22/15 - Move sub copyfiles to crsutils.pm # luoli 01/21/15 - Separate downgrade/deconfig flow # bbeelamk 01/08/15 - Fix lrg 14597150 # sbezawad 12/22/14 - Bug 20019354: Migrate OCR and OLR to new framework # xyuan 12/15/14 - Fix bug 20206616 # jmarcias 12/15/14 - Fix bug 14117160 # xyuan 12/10/14 - upgrade restructuring # muhe 12/10/14 - Fix bug 20193703,20193873 # muhe 12/10/14 - Modify sub upgrade_node # luoli 12/01/14 - Fix bug 20119857 # luoli 11/25/14 - Fix bug 19955755 for windows # xyuan 11/24/14 - Move sub get_oifcfg_info to crsutils.pm # bbeelamk 11/18/14 - upgrade from std cluster # luoli 11/14/14 - Fix bug 19955755 # muhe 11/04/14 - Fix bug 18844632 # xyuan 11/04/14 - Add orasrvm.pm # bbeelamk 11/03/14 - Fix bug 19849644 # muhe 10/23/14 - Fix bug 19854558 # bbeelamk 10/15/14 - Fix bugs 19688282,19674687 # luoli 10/14/14 - Fix bug 19539653 # jmarcias 09/30/14 - Fix bug 19640569 # luoli 09/27/14 - Fix bug 19684558 # bbeelamk 09/19/14 - Fix bug 19612597 # gnagiah 09/15/14 - Bug 19617592. handle node num 0 in case of vendor # clusterware # luoli 09/11/14 - Fix bug 19539418,19594906 # luoli 09/05/14 - Fix bug 19450633 # muhe 08/28/14 - Fix bug 19464978 # jmarcias 08/27/14 - Fix bug 19450419 # xyuan 08/19/14 - rsc modeling # muhe 08/15/14 - Fix bug 19444471 # rdasari 08/07/14 - add cvu on last node # lcarvall 07/23/14 - Bug 13800615 Use mnemonic string instead numeric values # rdasari 07/22/14 - start the stack to do asmca nonrolling upgrade # muhe 07/21/14 - Fix bug 19147315 # rdasari 07/15/14 - invoke startupgrade with no condition # xyuan 06/18/14 - Fix bug 19013444 # luoli 06/12/14 - Fix bug 18886094 # xyuan 06/06/14 - Remove the saved pfile for mgmtdb # rdasari 05/29/14 - modify 10.2 db resources bug 18708349 # spavan 05/13/14 - XbranchMerge spavan_bug-18648369 from st_has_12.1 # muhe 05/12/14 - Fix bug 18701111 # rdasari 05/09/14 - unset srvm_trace before srvctl config asm # luoli 05/07/14 - Fix bug 18667810 # xyuan 05/07/14 - Fix bug 18634842 # muhe 04/25/14 - Fix bug 18636678 # xyuan 04/24/14 - Fix bug 18635452 # luoli 04/23/14 - Fix bug 18643537 # rdasari 04/20/14 - create cred domains only for pre-12.1 upgrades # spavan 04/16/14 - fix bug18548752 - remove cvu earlier # xyuan 04/18/14 - Fix bug 18606913 # apfwkr 04/11/14 - Backport luoli_bug-18531995 from main # apfwkr 04/03/14 - Backport jgrout_bug-17815202 from main # luoli 03/31/14 - Fix lrg problem 11664878 # luoli 04/07/14 - Fix bug 18531995 # minzhu 04/04/14 - 18508267, install HAIP when upgrade from 11.1 # luoli 03/31/14 - Fix bug 18480923 # luoli 03/24/14 - Fix lrg problem 11664878 # jgrout 03/21/14 - Fix bug 17815202 # xyuan 03/12/14 - Fix bug 18377096 # xyuan 03/12/14 - Modify sub isFirstNodeToUpgrade # xyuan 03/03/14 - Fix bug 18279534 # xyuan 02/25/14 - Modified the command success criteria for # 'ocrconfig -manualbackup' # luoli 02/24/14 - Fix bug 17967087 # xyuan 02/24/14 - Consolidate Clusterware and ASM rolling migration # messages # xyuan 02/21/14 - Fix bug 18278616 # xyuan 02/16/14 - Fix bug 18128918 # xyuan 02/13/14 - On Windows, relocate mgmtdb to the installer node # luoli 02/07/14 - Fix bug 18161639 # xyuan 02/07/14 - Fix bug 18067685 # hmbui 02/06/14 - Improve KA Driver upgrade during CRS upgrade. # luoli 02/06/14 - Fix bug 18013254 and modify sub isFirstNodeToUpgrade # madoming 02/04/14 - Bug 18176819 Check isACFSPath after create adr dirs # samjo 01/29/14 - Bug 18147522. Error if cardinality not found # ssprasad 01/27/14 - bug 18102497:setup gpnp env to query from profile # shullur 01/26/14 - For fixing bug 17991840. # rdasari 01/23/14 - pass -skip with add nodeapps -u # luoli 01/20/14 - Modify sub configLastNode # xyuan 01/17/14 - Fix bug 17602658 # samjo 01/17/14 - Bug 18092855. Add checkpoint and -1 for ASM # cardinality # xyuan 01/15/14 - Fix bug 18080517 # luoli 01/13/14 - Ocr backup and restore # luoli 01/12/14 - Fix bug 18039176 # ssprasad 01/09/14 - Fix bug 17934142. Invoke AFD via osd_setup() # Call osd_setup() instead of s_osd_setup() # luoli 01/08/14 - Fix bug 17992711 # madoming 01/03/14 - Fix bug 17941243 # madoming 01/02/14 - Fix bug 17488446 # xyuan 12/29/13 - Fix bug 17984987 # luoli 12/26/13 - Fix bug 18004369 # xyuan 12/25/13 - Fix lrg problem 11122672 # xyuan 12/17/13 - Fix bug 17801136 # xyuan 12/17/13 - Fix bug 17947094 # xyuan 12/12/13 - Fix bug 17893782 # luoli 12/08/13 - Fix bug 17895028 # samjo 12/05/13 - Bug 17778985. Upgrading Flex ASM # loguzman 12/05/13 - Bug 16300127 Add all existent volumes to CRS # xyuan 12/04/13 - Fix bug 17899506 # xyuan 12/03/13 - Fix bug 17040372 # xyuan 11/24/13 - Fix bug 17778904 # madoming 11/22/13 - Fix bug 17841639 # luoli 11/21/13 - write ckpts in ocr # xyuan 11/20/13 - Fix bug 17786777 # jmunozn 11/19/13 - Fix bug 17778974 # bamorale 11/15/13 - Bug17054779 bring volume resources online # xyuan 11/14/13 - Fix bug 17763957 # luoli 11/11/13 - Fix bug 17762422 # xyuan 11/11/13 - Fix bug 17763351 # minzhu 11/08/13 - 15847266, haip in upgrade to 12.1 # madoming 11/08/13 - Add validation for current working directory # xyuan 10/31/13 - Fix bug 17700074 # xyuan 10/29/13 - Remove the invocation of createMgmtdbDir_Old # luoli 10/29/13 - Fix bug 17342639 # xyuan 10/28/13 - Fix bug 17293244 # xyuan 10/28/13 - Fix bug 17649114 # xyuan 10/23/13 - Fix bug 17433179 # jmunozn 10/21/13 - Fix bug 17618620 # xyuan 10/11/13 - Fix bug 17469910 # cnagur 09/24/13 - Support for TFA # xyuan 09/13/13 - Fix bug 17455758 # xyuan 09/10/13 - Fix bug 17412571 # xyuan 09/09/13 - Fix bug 17412464 # mkallana 08/28/13 - fix comments # xyuan 08/22/13 - Remove compilation warnings # mkallana 08/22/13 - call cssCleanIpc() # shmubeen 08/16/13 - Bug# 17262330 # samjo 08/13/13 - Use isCHASupported() # xyuan 08/08/13 - Query CRS software version using crsctl # instead of ocrdump # xyuan 07/24/13 - Starting from 12.1.0.1, call 'srvctl' from the new # home when stopping and deleting old listeners # xyuan 07/23/13 - Move sub isHomeShared to crsutils.pm # spavan 07/17/13 - fix bug16024563 - truncate values # xyuan 07/16/13 - Fix bug 17158147 # xyuan 07/14/13 - Fix bug 16912647 # shmubeen 07/08/13 - add support for AFD upgrade # xyuan 07/04/13 - Add checkpoint for upgrading CSS voting files # xyuan 06/26/13 - Delete all function (sub get101viphome) related # code. # samjo 06/19/13 - Bug 13108621. Add add_cha on last node # xyuan 06/12/13 - add start/success messages around major upgrade # operations # jgrout 06/11/13 - XbranchMerge jgrout_bug-16075946 from # st_has_12.1.0.1 # xyuan 05/29/13 - XbranchMerge xyuan_bug-16833151 from st_has_11.2.0.4 # xyuan 05/02/13 - Fix bug 16591778 # rdasari 04/18/13 - add scan change # xyuan 04/09/13 - Back up ocr.loc during upgrade # xyuan 03/29/13 - Fix bug 16559915 # xyuan 03/14/13 - Fix bug 16474419 # sidshank 02/26/13 - XbranchMerge sidshank_bug-16169568 from # st_has_12.1.0.1 # sidshank 02/18/13 - fix bug 16169568 # sidshank 02/13/13 - XbranchMerge sidshank_bug-16275273 from # st_has_12.1.0.1 # sidshank 02/04/13 - fix bug 16275273. # sidshank 01/16/13 - fix bug 16175611 # jgrout 01/08/13 - Fix bug 16075946 # xyuan 01/06/13 - XbranchMerge xyuan_bug-16026674 from st_has_12.1.0.1 # xyuan 12/25/12 - Fix bug 16014163 - disable listener changes for # Windows # xyuan 12/20/12 - Fix bug 16026674 # xyuan 12/13/12 - Fix bug 15994179 # xyuan 12/09/12 - Start listener for joining node # rdasari 11/14/12 - Fail if crsctl startupgrade fails # ssprasad 11/13/12 - Disable OKA actions for 12.1 # spavan 11/09/12 - fix bug15848256 - add checkpoints for CVU commands # on last node # xyuan 10/30/12 - Fix bug 14804296 - Call asmca -joinCluster on # joining node # xyuan 10/29/12 - Wait for CRSD to come up during the rerun of 'crsctl # startupgrade' # xyuan 10/17/12 - Fix bug 14648503 # spavan 10/17/12 - fix bug14771917 - store config info in check point # rdasari 10/17/12 - modify upgCreateCredDomain # spavan 10/09/12 - fix bug14739163 - isCVUConfigured should use old crs # home # xyuan 10/09/12 - Fix bug 14683232 # spavan 09/27/12 - fix bug14218734 - check cvu config # epineda 09/27/12 - Bugfix 13539130 # xyuan 09/18/12 - Fix bug 14603264 # xyuan 09/18/12 - Fix bug 14621340 - make error in # removing CVU fatal # shullur 09/17/12 - For adding check of version greater than 12 in case # of bdb # xyuan 09/13/12 - Fix bug 14603434 # jmunozn 09/11/12 - Execute addWallet_J2EEContainer() in first node when # not isOldVersionLT112() # xyuan 09/11/12 - Fix bug 14592705 # ssprasad 09/05/12 - Fix bug 14576870 - invoke OKA install # action in upgrade scripts. # dpham 08/21/12 - Add XAG component # rdasari 08/16/12 - write OLD_CSSD_LOGLEVEL in ckpt # xyuan 08/14/12 - Fix bug 14487626 # xyuan 08/08/12 - Fix bug 9584563 # xyuan 08/03/12 - Fix bug 14392140 # xyuan 08/01/12 - Fix bug 13730408 # xyuan 07/25/12 - Make few trace messages in preForceUpgChecks go to # the console # rdasari 07/20/12 - wait for stack start after set active version # rdasari 07/10/12 - do not add asm on a asm client cluster # rtamezd 07/02/12 - Use trace_lines() # xyuan 06/20/12 - Fix bug 14202360 # xyuan 06/20/12 - Fix bug 14026786 # shullur 06/15/12 - For handling default case of bdb location # rdasari 06/02/12 - change default to root automation for Windows # sidshank 05/23/12 - fix bug 14106919 # sidshank 05/22/12 - fix bug 14099962 # xyuan 05/16/12 - Cache the value for # isFirstNodeToUpgrade/isLastNodeToUpgrade # shullur 05/09/12 - For fixing the exists in case of failure of copy of # orafiles # sidshank 05/03/12 - remove s_redirect/restore output subroutines # xyuan 05/01/12 - Remove modifyClusterName because the installer will # instantiate the CLUSTER_NAME in crsconfig_params # after the bug 12681659 is fixed # sidshank 04/25/12 - fix for bug 14004260 # xyuan 04/24/12 - Implementation for adding the local node to an # upgraded cluster # xyuan 04/24/12 - Remove asmca invocation on the last node # rdasari 04/19/12 - bounce ohasd after creating ohasd resources # gmaldona 03/16/12 - adding oc4j setup # rtamezd 03/05/12 - Fix bug 13803068 # rtamezd 02/28/12 - Fix bug 13401742 # akhaladk 02/28/12 - Bug 13785751 # xyuan 02/24/12 - Fix bug 13770508 # xyuan 02/14/12 - Do not change permissions on EXTERNAL_ORACLE if .ssh # exists under it during upgrade # xyuan 02/09/12 - Fix bug 13612321 & 12710059 # sidshank 02/09/12 - Remove call to first_node_tasks() # rtamezd 02/07/12 - Move upgradeModelResType to crsutils.pm # xyuan 02/02/12 - Force upgrade support # xyuan 02/01/12 - Remove old ASM files when upgrading from # 11.2.0.1/11.2.0.2 # xyuan 01/30/12 - Create the credential domains for ASM and Gridhome # during upgrade # rsirigan 01/27/12 - export upgrade_OCR for pcw_upgrade_nodes macro # xyuan 01/12/12 - Fix bug 13426221 # shullur 01/06/12 - XbranchMerge shullur_bug-12976590 from st_has_11.2.0 # spavan 01/06/12 - fix bug 13401742 # xyuan 12/25/11 - Fix bug 13532044 # gsanders 12/23/11 - Eliminate ACFS registry. Check upgrade_acfs_registry ret # rtamezd 12/21/11 - Updated call to add_GNS # shullur 12/12/11 - XbranchMerge shullur_bug-12673460 from st_has_11.2.0 # xyuan 12/13/11 - Removed unnecessary comments # xyuan 12/10/11 - Fix bug 13435658 # xyuan 12/06/11 - Fix bug 13435843 # xyuan 12/04/11 - Fix bug 13091719 # agraves 11/30/11 - Update reboot necessary mechanism to be a failure, # not a continuation and a suggestion. # shullur 11/03/11 - XbranchMerge shullur_bug-12614795 from st_has_11.2.0 # shullur 11/03/11 - XbranchMerge shullur_bug_11852891 from st_has_11.2.0 # rvadraha 10/25/11 - Bug13247694, Fix ocfs upgrade # xyuan 11/14/11 - Fix bug 13340630 # xyuan 11/07/11 - Forward merge fixes for bug 12344535 & 12640551 # shullur 11/03/11 - XbranchMerge shullur_bug_11852891 from st_has_11.2.0 # xyuan 10/31/11 - Fix bug 13328777 # xyuan 09/29/11 - Forward merge fix for bug 12640884 # xyuan 09/20/11 - Fix bug 12737227 # priagraw 08/19/11 - remove call to clsfmt # xyuan 08/04/11 - XbranchMerge xyuan_bug-12795595 from # st_has_11.2.0.3.0 # xyuan 08/03/11 - XbranchMerge xyuan_bug-12698968 from # st_has_11.2.0.3.0 # xesquive 07/26/11 - Add functions set_bold and reset_bold instead of # print color # xyuan 07/25/11 - Rename create_crs_resources to # create_ohasd_resources # rdasari 07/15/11 - do not use avlookup for siha # lmortime 06/27/11 - Make EVMD start first # spavan 06/19/11 - remove and add cvu in upgrade # dpham 05/23/11 - Modulize upgrade function # rdasari 05/20/11 - remove ocr check from preUpgradeChecks # dpham 03/28/11 - New for 12c # package crsupgrade; use strict; use English; use File::Copy; use File::Path; use File::Find; use File::Basename; use File::Spec::Functions; use Sys::Hostname; use Carp; use Socket; use Cwd; use Env qw(NLS_LANG); use Env qw(SRVM_TRACE); use Env qw(ORACLE_HOME); no if $] >= 5.017011, warnings => 'experimental::smartmatch'; # root script modules use crsinstall; use crsutils; use crsgpnp; use oracss; use oraacfs; use oraafd; use s_crsutils; use crstfa; use oraClusterwareComp; use oraolr; use oraocr; use s_oraolr; use s_oraocr; use oraasm; use oraohasd; use oraqosmserver; use orasrvm; use crska; use crscpcfg; use crsxag; use oraios; use Exporter; use vars qw(@ISA @EXPORT @EXPORT_OK); @ISA = qw(Exporter); my @exp_func = qw(get_oldconfig_info); push @EXPORT, @exp_func; my @force_upgrade_nodes; sub new { shift; crsutils->new(@_); rscPreChecks(); # upgrade if ($CFG->SIHA) { if ($CFG->ORACLERESTART) { OAUpgrade(); } else { # Deprecated HAUpgrade(); } } else { CRSUpgrade(); } } my @CRS_UPGRADE_STAGES = ( {"name" => "SetupEnv", "checkpoint" => "null", "sub" => \&crs_upgrade_init}, {"name" => "UpgradeTFA", "checkpoint" => "null", "sub" => \&upgrade_tfa}, {"name" => "ValidateEnv", "checkpoint" => "null", "sub" => \&crs_upgrade_validate}, {"name" => "GetOldConfig", "checkpoint" => "ROOTCRS_OLDHOMEINFO", "sub" => \&get_oldconfig_info}, {"name" => "GenSiteGUIDs", "checkpoint" => "null", "sub" => \&gen_site_GUIDs}, {"name" => "UpgPrechecks", "checkpoint" => "ROOTCRS_UPGPRECHECKS", "sub" => \&crs_upgrade_prechecks}, {"name" => "SaveParamFile", "checkpoint" => "ROOTCRS_PARAM", "sub" => \&save_param_file}, {"name" => "SetupOSD", "checkpoint" => "null", "sub" => \&setup_actions}, {"name" => "PreUpgrade", "checkpoint" => "null", "sub" => \&preupgrade_actions}, {"name" => "CheckCRSConfig", "checkpoint" => "null", "sub" => \&check_crs_config}, {"name" => "UpgradeOLR", "checkpoint" => "ROOTCRS_OLR", "sub" => \&crs_upgrade_olr}, {"name" => "ConfigCHMOS", "checkpoint" => "null", "sub" => \&create_CHMConfig}, {"name" => "UpgradeAFD", "checkpoint" => "ROOTCRS_AFDINST", "sub" => \&upgrade_AFD}, {"name" => "createOHASD", "checkpoint" => "ROOTCRS_OHASD", "sub" => \&create_ohasd}, {"name" => "ConfigOHASD", "checkpoint" => "ROOTCRS_INITRES", "sub" => \&config_ohasd}, {"name" => "InstallACFS", "checkpoint" => "ROOTCRS_ACFSINST", "sub" => \&install_ACFS}, {"name" => "InstallKA", "checkpoint" => "ROOTCRS_OKAINST", "sub" => \&install_KA}, {"name" => "UpgradeCluster", "checkpoint" => "null", "sub" => \&upgrade_cluster}, {"name" => "UpgradeNode", "checkpoint" => "ROOTCRS_NODECONFIG", "sub" => \&crs_upgrade_node}, {"name" => "PostUpgrade", "checkpoint" => "null", "sub" => \&postupgrade_actions} ); my @SANDBOX_ENV_VARS_SET = ("OLR_LOC", "OCR_LOC", "OCR_LOC_FILE", "OCR_DEVELOPER_ENV", "ORACLE_HOME", "CSS_CLUSTERNAME", "_SCLS_DEVELOPMENT", "HAS_DEVELOPMENT_ENVIRONMENT", "T_HAS_WORK"); sub crs_upgrade_init { # validate RAC_ON/RAC_OFF if (! is_dev_env () && ! isRAC_appropriate()) { exit 1; } $CFG->compAFD(oraClusterwareComp::oraafd->new("AFD")); $CFG->compACFS(oraClusterwareComp::oraacfs->new("ACFS")); # instantiate scripts instantiate_scripts(); # Initialize oraolr and oraocr modules $CFG->compOLR(oraClusterwareComp::oraolr->new("OLR")); $CFG->compOCR(oraClusterwareComp::oraocr->new("OCR")); $CFG->compCHM(oraClusterwareComp::orachm->new("CHM")); # Initialize oraios module $CFG->compIOS(oraClusterwareComp::oraios->new("IOS")); # run directory creation, files creation/permissions modules setup_env(($CFG->isRerun) ? FALSE : TRUE); ($CFG->compAFD)->preUpgradeCheck(); ($CFG->compACFS)->preUpgradeCheck(); my $ckpt_downgrade = "ROOTCRS_DOWNGRADE"; my $ckpt_lastnode = "ROOTCRS_LASTNODE"; if (isCkptSuccess($ckpt_downgrade)) { trace("Upgrading after a successful prior downgrade"); # To-do: a message might be needed. trace("Make sure mgmtdb has been reconfigured manually on the lower version". "before proceeding to another upgrade."); remove_checkpoints(); } create_starting_global_ckpt_entry(); create_starting_ckpt_entry(); if (isFirstNodeToUpgrade()) { trace("Perform upgrade on the first node"); $CFG->nodeAttributeUpgrade(FIRST_NODE_TO_UPGRADE); } elsif ($CFG->isLastNodeToUpgrade) { trace("Perform upgrade on the last node"); $CFG->nodeAttributeUpgrade(LAST_NODE_TO_UPGRADE); } else { trace("Perform upgrade on the middle node"); $CFG->nodeAttributeUpgrade(MIDDLE_NODE_TO_UPGRADE); } $CFG->compASM(oraClusterwareComp::oraasm->new("ASM")); $CFG->compSRVM(oraClusterwareComp::orasrvm->new("SRVM")); $CFG->compOHASD(oraClusterwareComp::oraohasd->new("OHASD")); } sub upgrade_tfa { #Patch TFA patch_tfa("1", "2"); } sub crs_upgrade_validate { if ($CFG->JOIN) { pullGlobalCkptFileNIndex(); } else { checkFirstNodeStatus(); } if (isFirstNodeToUpgrade()) { my $ckptName = "ROOTCRS_OCRSTATUS"; if (! isCkptexist($ckptName)) { trace("Writing checkpoint for OCR status."); writeCkpt($ckptName, CKPTSTART); } } } sub crs_upgrade_prechecks { # The force upgrade checks should always be executed # outside the ckpt. if (($CFG->FORCE) && (FAILED == preForceupgChecks())) { die(dieformat(429)); } my $ckptName = "ROOTCRS_UPGPRECHECKS"; my $ckptStatus = CKPTSTART; my $wipCkpt = $CFG->wipCkptName; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); } if ($ckptStatus eq CKPTSUC) { trace("Upgrade prechecks have passed."); return; } else { $CFG->wipCkptName($ckptName); writeCkpt($ckptName, CKPTSTART); } precheck(); preUpgChecks(); my $oraolr = $CFG->compOLR; my $oraocr = $CFG->compOCR; my $oraasm = $CFG->compASM; my $orasrvm = $CFG->compSRVM; my $oraohasd = $CFG->compOHASD; # global checks of all components for upgrade if (isFirstNodeToUpgrade()) { $oraolr->preUpgradeCheck() || die(dieformat(362)); $oraocr->preUpgradeCheck() || die(dieformat(362)); $oraasm->preUpgradeCheck() || die(dieformat(362)); $orasrvm->preUpgradeCheck() || die(dieformat(362)); $oraohasd->preUpgradeCheck() || die(dieformat(362)); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName($wipCkpt); } sub setup_actions { osd_setup(); if ($CFG->ORACLERESTART) { set_perms_ocr_vdisk(); } } sub preupgrade_actions { trace("Preparing to upgrade Clusterware ..."); perform_crs_pre_upgrade(); # Add the DB service sids from the ORA_ASMDBA group to the # ORA_ASMADMIN group. Should be called with the databases # and the old stack if (($CFG->platform_family eq "windows") && (!isOldVersionLT121())) { s_migrateDBServiceSidsUpgrade(); } } sub check_crs_config { # Check if CRS is already configured my $crsConfigured = FALSE; my $gpnp_setup_type = GPNP_SETUP_BAD; # Initialize the CHM module $CFG->compCHM(oraClusterwareComp::orachm->new("CHM")); my $orachm = $CFG->compCHM; check_CRSConfig($CFG->ORA_CRS_HOME, $CFG->HOST, $CFG->params('ORACLE_OWNER'), $CFG->params('ORA_DBA_GROUP'), $CFG->params('GPNPGCONFIGDIR'), $CFG->params('GPNPCONFIGDIR'), $crsConfigured, $gpnp_setup_type) || die(dieformat(315)); # Verify bdb location. Initially its in CRS HOME. oclumon changes later if ($orachm->isSupported() && !is_dev_env() && !$orachm->isReposBDB()) { my $nodelist = $CFG->oldconfig('NODENAME_LIST'); my $bdbloc = catfile($CFG->ORA_CRS_HOME, 'crf', 'db', $CFG->HOST); $orachm->chm_check_bdbloc($bdbloc, $nodelist); } # bug#17335617 # Clean up CSS IPC files created by old version stack. trace ("Cleaning up CSS Ipc files!"); if (! CSS_Clean_Ipc_files()) { trace ("Clean up CSS Ipc files failed"); } } sub crs_upgrade_olr { # If it's the Flex cluster, gets the local node role and saves it before # calling upgrade_olr_config() as 'clscfg -localupgrade' will remove # the key 'SYSTEM.OHASD' from OLR. # As a consequence, the key [SYSTEM.OHASD.RESOURCES.ora!cssd.CONFIG] # is missing after upgrade_olr_config(). if (! isOldVersionLT121()) { trace("Get the local node role and restore it for Flex cluster upgrades"); get_localNodeRole(); } # Populates the cluster properties in the global variables getClusterProperties(); # Check existing configuration, Create and populate OLR and OCR keys upgrade_olr_config() || die(dieformat(316)); } sub install_AFD { # install AF driver, Do it much before than anything else perform_installAFDriver(); } sub create_ohasd { # Setup OHASD service/daemon perform_register_ohasd(); } sub config_ohasd { # Start OHASD service/daemon perform_start_ohasd() || die(dieformat(318)); # create OHASD resources create_ohasd_resources(); # update OHASD resources with the saved attributes from the old home if ($CFG->compOHASD->updateResAttr() == FAILED) { die(dieformat(713)); } # create rootcert domain in OLR and store the certificate; dies on failure create_rootcert_domain(OLR) if (!exist_rootcert_domain(OLR)); store_rootcert_olr(); } sub upgrade_AFD { # install AFD driver, do it much before anything else $CFG->compAFD->upgradeCurrentNode(); } sub install_ACFS { # install USM driver my $oraacfs = $CFG->compACFS; $oraacfs->upgradeCurrentNode($oraacfs->UPGRADE_ACFS); } sub install_KA { # install KA driver. Supported only in cluster. perform_installKADriver(); } sub crs_upgrade_node { upgrade_node(); if (isFirstNodeToUpgrade()) { get_cluster_info(); } } sub postupgrade_actions { removeDiagsnap() if (isFirstNodeToUpgrade()); ModActionScript(); $CFG->compAFD->postUpgradeCurrentNode(); $CFG->compASM->postUpgradeCurrentNode(); $CFG->compOLR->postUpgradeCurrentNode(); $CFG->compOCR->postUpgradeCurrentNode(); # Configure IOS only where it is supported if ($CFG->compIOS->isSupported()) { $CFG->compIOS->postUpgradeCurrentNode(); } configureCvuRpm(); install_xag($CFG->ORA_CRS_HOME); cleanASMFiles(); createMgmtdbDir(); if (($CFG->platform_family eq "unix") && ($CFG->JOIN)) { trace("Starting listeners on node that is joining the cluster ..."); if (! startListeners(TRUE)) { exit(1); } } my $oraacfs = $CFG->compACFS; $oraacfs->postUpgradeCurrentNode(); # Update history on last node to upgrade if (isLastNodeToUpgrade()) { my $oraocr = $CFG->compOCR; # Upgrade is completely only when the upgrade end marker is set by # installer. from and to version/patchlevel information is not available # by the API that sets the upgrade end marker. Set a temp record with the # required information and the API that sets the end marker modifies it and # writes the final record my $type = ($CFG->FORCE ? "ForceUpgrade_Temp" : "Upgrade_Temp"); my $fromVersion = getcrsrelver($CFG->OLD_CRS_HOME); # No patchlevel if lower version is pre-12.1. Default to 0. my $fromPatch = 0; if (!isOldVersionLT121()) { $fromPatch = getReleasePatch($CFG->OLD_CRS_HOME); } my $toVersion = getcrsrelver($CFG->ORA_CRS_HOME); my $toPatch = getReleasePatch($CFG->ORA_CRS_HOME); $oraocr->updateHistory($type, $fromVersion, $fromPatch, $toVersion, $toPatch, $CFG->ORA_CRS_HOME); } } sub CRSUpgrade { my $count = 0; foreach my $stage (@CRS_UPGRADE_STAGES) { my $name = $stage->{"name"}; my $checkpoint = $stage->{"checkpoint"}; my $func = $stage->{"sub"}; trace("Executing the [$name] step with checkpoint [$checkpoint] ..."); if ($name eq "UpgradeTFA" && isOnlyDryRun()) { trace("Skipping TFA because CHECKCRSENTITIES option is ON"); $count++; next; } # We don't print the first step as clsecho might not be available # so to keep the steps consistent the first step is never printed. if (0 != $count && !isOnlyDryRun()) #no msg for upgrade dry run { print_info(595, $count, scalar(@CRS_UPGRADE_STAGES) - 1, $name); } &$func(); $count++; } if ($count == scalar(@CRS_UPGRADE_STAGES)) { writeCkpt("ROOTCRS_STACK", CKPTSUC); set_bold(); print_info(325); reset_bold(); if (1 == $CFG->SUCC_REBOOT) { set_bold(); print_info(411); reset_bold(); } } else { set_bold(); print_error(326); reset_bold(); exit 100; } } my @HA_UPGRADE_STAGES = ( {"name" => "SetupEnv", "checkpoint" => "null", "sub" => \&ha_upgrade_init}, {"name" => "UpgPrechecks", "checkpoint" => "null", "sub" => \&ha_upgrade_prechecks}, {"name" => "GetOldConfig", "checkpoint" => "ROOTHAS_OLDHOMEINFO", "sub" => \&get_siha_oldconfig_info}, {"name" => "GenSiteGUIDs", "checkpoint" => "null", "sub" => \&gen_site_GUIDs}, {"name" => "SetupOSD", "checkpoint" => "null", "sub" => \&setup_actions}, {"name" => "PreUpgrade", "checkpoint" => "ROOTHAS_UPGRADEASM", "sub" => \&prepare_to_upgrade_siha}, {"name" => "UpgradeAFD", "checkpoint" => "ROOTHAS_AFDINST", "sub" => \&upgrade_AFD}, {"name" => "UpgradeOLR", "checkpoint" => "ROOTHAS_OLR", "sub" => \&ha_upgrade_olr}, {"name" => "UpgradeOCR", "checkpoint" => "ROOTHAS_LOCOCR", "sub" => \&ha_upgrade_ocr}, {"name" => "CreateOHASD", "checkpoint" => "ROOTHAS_OHASD", "sub" => \&create_ohasd}, {"name" => "ConfigOHASD", "checkpoint" => "ROOTHAS_INITRES", "sub" => \&ha_config_ohasd}, {"name" => "UpgradeSIHA", "checkpoint" => "ROOTHAS_NODECONFIG", "sub" => \&upgrade_siha}, {"name" => "InstallACFS", "checkpoint" => "ROOTHAS_ACFSINST", "sub" => \&ha_install_ACFS}, ); sub HAUpgrade { my $count = 0; foreach my $stage (@HA_UPGRADE_STAGES) { my $name = $stage->{"name"}; my $checkpoint = $stage->{"checkpoint"}; my $func = $stage->{"sub"}; trace("Executing the [$name] step with checkpoint [$checkpoint] ..."); # We don't print the first step as clsecho might not be available # so to keep the steps consistent the first step is never printed. if (0 != $count) { print_info(595, $count, scalar(@HA_UPGRADE_STAGES) - 1, $name); } &$func(); $count++; } if ($count == scalar(@HA_UPGRADE_STAGES)) { writeCkpt("ROOTHAS_STACK", CKPTSUC); set_bold(); print_info(327); reset_bold(); } else { set_bold(); print_error(596); reset_bold(); exit 100; } } my @HA_UPGRADE_TO_OA_STAGES = ( {"name" => "SetupEnv", "checkpoint" => "null", "sub" => \&ha_upgrade_init}, {"name" => "UpgPrechecks", "checkpoint" => "null", "sub" => \&ha_upgrade_prechecks}, {"name" => "GenSiteGUIDs", "checkpoint" => "null", "sub" => \&gen_site_GUIDs}, {"name" => "SaveParamFile", "checkpoint" => "ROOTCRS_PARAM", "sub" => \&save_param_file}, {"name" => "SetupOSD", "checkpoint" => "null", "sub" => \&setup_actions}, {"name" => "PreUpgrade", "checkpoint" => "null", "sub" => \&prepare_to_upgrade_siha}, {"name" => "CheckCRSConfig", "checkpoint" => "null", "sub" => \&check_crs_config}, {"name" => "SetupLocalGPNP", "checkpoint" => "ROOTCRS_GPNPSETUP", "sub" => \&setup_local_gpnp}, {"name" => "CreateRootCert", "checkpoint" => "null", "sub" => \&create_root_cert}, {"name" => "UpgradeOLR", "checkpoint" => "ROOTCRS_OLR", "sub" => \&ha_upgrade_olr}, {"name" => "CreateOHASD", "checkpoint" => "ROOTCRS_OHASD", "sub" => \&create_ohasd}, {"name" => "ConfigOHASD", "checkpoint" => "ROOTCRS_INITRES", "sub" => \&config_ohasd}, {"name" => "InstallAFD", "checkpoint" => "ROOTCRS_AFDINST", "sub" => \&install_AFD}, {"name" => "InstallACFS", "checkpoint" => "ROOTCRS_ACFSINST", "sub" => \&install_ACFS}, {"name" => "InstallKA", "checkpoint" => "ROOTCRS_OKAINST", "sub" => \&install_KA}, {"name" => "InitConfig", "checkpoint" => "ROOTCRS_BOOTCFG", "sub" => \&init_config}, {"name" => "StartCluster", "checkpoint" => "null", "sub" => \&start_cluster}, {"name" => "ConfigNode", "checkpoint" => "ROOTCRS_NODECONFIG","sub" => \&config_node}, {"name" => "PostConfig", "checkpoint" => "null", "sub" => \&postconfig_actions} ); sub OAUpgrade { my $count = 0; foreach my $stage (@HA_UPGRADE_TO_OA_STAGES) { my $name = $stage->{"name"}; my $checkpoint = $stage->{"checkpoint"}; my $func = $stage->{"sub"}; trace("Executing the [$name] step with checkpoint [$checkpoint] ..."); # We don't print the first step as clsecho might not be available # so to keep the steps consistent the first step is never printed. if (0 != $count) { print_info(595, $count, scalar(@HA_UPGRADE_TO_OA_STAGES)-1, $name); } &$func(); $count++; } if ($count == scalar(@HA_UPGRADE_TO_OA_STAGES)) { writeCkpt("ROOTCRS_STACK", CKPTSUC); set_bold(); print_info(698); reset_bold(); } else { set_bold(); print_error(699); reset_bold(); exit 100; } } sub ha_upgrade_init { # validate RAC_ON/RAC_OFF if (! is_dev_env () && (! isRAC_appropriate())) { exit 1; } $CFG->compAFD(oraClusterwareComp::oraafd->new("AFD")); $CFG->compACFS(oraClusterwareComp::oraacfs->new("ACFS")); # instantiate scripts instantiate_scripts(); # Initialize oraolr and oraocr modules $CFG->compOLR(oraClusterwareComp::oraolr->new("OLR")); $CFG->compOCR(oraClusterwareComp::oraocr->new("OCR")); # run directory creation, files creation/permissions modules setup_env(($CFG->isRerun) ? FALSE : TRUE); if ($CFG->ORACLERESTART) { $CFG->nodeAttributeUpgrade(SIHA_NODE_UPGRADE_TO_OA); $CFG->params('NETWORKS', OR_NETWORKS); trace("NETWORK setting of Oracle Autostart:"); trace("NETWORKS=" . $CFG->params('NETWORKS')); } else { # Deprecated $CFG->nodeAttributeUpgrade(SIHA_NODE_TO_UPGRADE); } $CFG->compASM(oraClusterwareComp::oraasm->new("ASM")); $CFG->compOHASD(oraClusterwareComp::oraohasd->new("OHASD")); $CFG->compSRVM(oraClusterwareComp::orasrvm->new("SRVM")); } sub ha_upgrade_prechecks { create_starting_ckpt_entry(); precheck_siha(); $CFG->compAFD->preUpgradeCheck(); $CFG->compACFS->preUpgradeCheck(); } sub ha_upgrade_olr { my $oraolr = $CFG->compOLR; my $wipCkpt = (($CFG->SIHA && ! $CFG->ORACLERESTART) ? "ROOTHAS_STACK" : "ROOTCRS_STACK"); my $ckptName = (($CFG->SIHA && ! $CFG->ORACLERESTART) ? "ROOTHAS_OLR" : "ROOTCRS_OLR"); if ($CFG->ORACLERESTART) { # Populates the cluster properties in the global variables getClusterProperties(); } if (isOLRConfiguredCkpt()) { return SUCCESS; } if ($CFG->ORACLERESTART) { my $oraocr = $CFG->compOCR; $oraocr->upgradeCurrentNode($oraocr->INITIALIZE_OCRLOC_FILE); $oraolr->upgradeCurrentNode($oraolr->UPDATE_OLRLOC); initial_cluster_validation(); $oraolr->upgradeCurrentNode($oraolr->CONFIGURE_OLR); writeCkpt($ckptName, CKPTSUC); trace("OLR initialization - successful"); return SUCCESS; } else { # Upgrade OLR for SIHA $oraolr->upgradeCurrentNode(); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName($wipCkpt); trace("OLR initialization - successful"); if ($CFG->ORACLERESTART) { # Initialize the settings in the SCRBASE s_init_scr (); } } sub ha_config_ohasd { # Delete and recreate ohasd resources my $del_ohasd_res = TRUE; my $status; my $oraohasd = $CFG->compOHASD; # Need to start OHASD as non-$SUPERUSER, i.e., as crsuser, which in this case # would be ORACLE_OWNER trace ("Starting ohasd"); start_siha_ohasd(); # Check if the service/daemon has started trace ("Checking ohasd"); my $ohasd_running = check_service ("ohasd", 360); if ($ohasd_running) { trace ("ohasd started successfully"); } else { die(dieformat(199)); } trace ("Creating HA resources and dependencies"); my $ckptName = "ROOTHAS_INITRES"; if ($del_ohasd_res && (getCkptStatus($ckptName) ne CKPTSUC)) { my $crsctl = crs_exec_path('crsctl'); # There should not be crs and ctss in SIHA, # so remove both of them system_cmd_capture($crsctl, "delete", "type", "ora.crs.type", "-init"); system_cmd_capture($crsctl, "delete", "type", "ora.ctss.type", "-init"); $status = $oraohasd->cleanupOHASDResources(); } $status = perform_configure_hasd(); if ($status) { trace ("Successfully created OHASD resources for Oracle Restart"); } else { die(dieformat(200)); } } sub upgrade_siha { # start checkpoint my $success; my $ckptStatus; my $ckptName = "ROOTHAS_NODECONFIG"; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); trace("'$ckptName' state is $ckptStatus"); if (($ckptStatus eq CKPTSUC) && (! $CFG->FORCE)) { trace("HA node resources are already configured"); $CFG->wipCkptName("ROOTHAS_STACK"); return SUCCESS; } } writeCkpt($ckptName, CKPTSTART); $CFG->wipCkptName($ckptName); # if old SI CSS was running when roothas.pl was invoked *or* ASM needs to be # upgraded, # restart the stack so that ohasd starts all autostart # resources (ASM, DB). CSS gets started as a dependency here. # CSS needs to be up for ASMCA to configure/start the new ASM. if ($CFG->restart_css || ($CFG->params('ASM_UPGRADE') =~ m/true/i)) { # No need to enable CSSD and DISKMON for upgrades # from 12.1.0.1 and later. if (isOldVersionLT121()) { trace("Enabling ora.cssd and ora.diskmon ..."); # Enable ora.cssd my $crsctl = catfile($CFG->ORA_CRS_HOME, 'bin', 'crsctl'); my @cmd = ($crsctl, 'modify', 'res', 'ora.cssd', '-attr', "\"ENABLED=1\"", '-init'); my $status = system_cmd(@cmd); if (0 != $status) { writeCkpt($ckptName, CKPTFAIL); die("Failed to enable CSSD startup"); } if ($CFG->platform_family ne "windows") { # Enable ora.diskmon @cmd = ($crsctl, 'modify', 'res', 'ora.diskmon', '-attr', "\"ENABLED=1\"", '-init'); $status = system_cmd(@cmd); if (0 != $status) { writeCkpt($ckptName, CKPTFAIL); die("Failed to enable DISKMON startup"); } } } trace("Restarting the stack"); # restart the ohasd so that all its managed resources start with the new profiles stopOracleRestart() || die(dieformat(348)); startOhasdOnlyInSIHA() || die(dieformat(318)); } # Invoke upgrademodel for SIHA $success = upgsihamodel(); # add ons by default for SIHA if (! isONSexist()) { $success = add_ons(); } # start evmd in SIHA mode to replace eonsd functionality if (!start_resource("ora.evmd", "-init")) { die(dieformat(202)); } if($success) { writeCkpt($ckptName, CKPTSUC); } else { writeCkpt($ckptName, CKPTFAIL); } # backup OLR my $ocrconfig = catfile($CFG->params('ORACLE_HOME'), 'bin', 'ocrconfig'); my $rc = system ("$ocrconfig -local -manualbackup"); if ($rc == 0) { trace ("$ocrconfig -local -manualbackup ... passed"); } else { trace ("$ocrconfig -local -manualbackup ... failed"); } cleanASMFiles(); } sub ha_install_ACFS { my $oraacfs = $CFG->compACFS; # Not sure why OHASD need to be up even CKPT is succeed? if (!isCkptSuccess("ROOTHAS_ACFSINST")) { # Only ohasd needs to be up before installing acfs drivers stopOracleRestart() || die(dieformat(348)); startOhasdOnlyInSIHA() || die(dieformat(318)); } $oraacfs->upgradeCurrentNode($oraacfs->UPGRADE_ACFS); } sub get_localNodeRole { trace("Get the role of the local node"); my $role = $CFG->oldconfig('NODE_CONFIG_ROLE'); trace("Save the role for the local node: $role"); $CFG->localNodeRole($role); } sub upgrade_cluster { my $oraocr = $CFG->compOCR; # Import ASM credentials into OLR if the lower version # GI is configured with the legacy ASM if ((isLegacyASM()) && (! isFirstNodeToUpgrade())) { if ($CFG->JOIN) { # If asm credential file not exists, we need pull # asm credential file from any of live nodes. my $status = pull_asm_cred_file($CFG->params('NODE_NAME_LIST')); if (SUCCESS != $status) { die(dieformat(442)); } } trace("Importing ASM credentials into OLR ..."); import_asm_credentials(); } my $ocrOnlymode = FALSE; if ((isLegacyASM()) && (isFirstNodeToUpgrade())) { # Bug 24287160: avoid disabling the resources on rerun. my $ckptName = "ROOTCRS_RESOURCEDISABLED"; my $wipCkpt = $CFG->wipCkptName; if (! isCkptSuccess($ckptName)) { $CFG->wipCkptName($ckptName); writeCkpt($ckptName, CKPTSTART); if(isOldVersionLT121()) { my $crsctl = crs_exec_path('crsctl'); my @out = system_cmd_capture($crsctl, "modify", "resource", "ora.crsd", "-init", "-attr", '"START_MODE=ocr"'); my $status = shift @out; trace("The command $crsctl modify resource ora.crsd". " -init -attr \"START_MODE=ocr\" returned wih $status"); if ($status != 0) { trace("\"crsctl modify resource ora.crsd -init -attr\" failed with status $status"); print_lines(@out); die(dieformat(180, 'crsctl modify resource ora.crsd -init -attr')); } } else { my $crshome = $CFG->ORA_CRS_HOME; trace("Disabling the crs resources"); setCRSResourceUseAttr($crshome, 'disable') || die(dieformat(180, 'crsctl set resource use')); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName($wipCkpt); } $ocrOnlymode = TRUE; } set_logging(); if (isFirstNodeToUpgrade()) { set_flag_for_ocr_changed(); } if (GI_STACK_UP != checkGIStack($CFG->ORA_CRS_HOME)) { if(isOldVersionLT121()) { stopFullStack("force") || die(dieformat(349)); startOhasdOnly() || die(dieformat(117)); startStackFor112Upg(); if (!$ocrOnlymode) { if (!check_service("crs", 120)) { die(dieformat(251)); } print_info(343); } } else # old version is 121 or above { # Start CRS with '-wait' option to wait for the autostart to complete trace("Old version is 12.1 or above, start CRS with '-wait' option."); stopFullStack("force") || die(dieformat(349)); startFullStack($CFG->ORA_CRS_HOME) || die(dieformat(117)); if (!$ocrOnlymode) { print_info(343); } } } # copy OCR backup to shared storage if OCR backup file is generated on # OCR master node, which may not be the first node. trace("Start to copy OCR manual backup file to shared storage if OCR " . "backup file is generated on OCR master node."); $oraocr->copyOcrBackupToSharedStorage(); if ($CFG->isFirstNodeToUpgrade) { # get the OCR backup file name and store it into global ckpt $oraocr->getOcrbackupFileNameOnDG(); } upgrade_clscfg(); upgrade_asm_credentials() if (isFirstNodeToUpgrade()); # create rootcert domain in OCR and store the certificate; dies on failure if ($CFG->isFirstNodeToUpgrade) { create_rootcert_domain(OCR) if (!exist_rootcert_domain(OCR)); store_rootcert_ocr(); } } sub startStackFor112Upg { my $orachm = $CFG->compCHM; trace("Old version is lower than 12.1, start CRS resources one by one."); # Start EVM if (! start_resource("ora.evmd", "-init")) { print_error(117); die(dieformat(250)); } # Start gpnpd if (! start_clusterware(START_STACK_GPNPD)) { print_error(117); die(dieformat(242)); } # Start CSS in clustered mode if (! CSS_start_clustered()) { print_error(117); die(dieformat(244)); } # Start CTSS with reboot option to signal step sync # Note: Before migrating stack startup to 'crsctl start crs', # 'CTSS_REBOOT=TRUE' is a workaround to signal step sync. if (! start_resource("ora.ctssd", "-init", "-env", "USR_ORA_ENV=CTSS_REBOOT=TRUE")) { print_error(117); die(dieformat(245)); } # Start CRF if (! $orachm->isSupported() || ! start_resource("ora.crf", "-init")) { trace ("start_clster: Failed to start CRF"); # We don't want IPD failure to stop rest of the stack # from coming up. So, no exit here! } # startup the HAIP if it was enabled before upgrade # do not start HAIP when upgrading Exadata from 11.2 to 12.2 - bug 24578492 if (!s_is_Exadata() && $CFG->OLD_HAIP_ENABLED == 1) { start_HAIP(); } # Start if this is Hub node and ocr/vd is on ASM my $localNodeRole = getNodeRoleStatus(); trace("The active role of local node is $localNodeRole"); if ((isOCRonASM()) && (NODE_ROLE_HUB eq $localNodeRole)) { start_asm_resource(); } # Start CRS if (!start_resource("ora.crsd", "-init")) { print_error(117); die(dieformat(249)); } } # This subroutine is only used during upgrade sub start_asm_resource { my $crsHome = getCrsHome(); my $CRSCTL = catfile ($crsHome, "bin", "crsctl"); my @cmd = ($CRSCTL, 'start', 'resource', 'ora.asm', '-init'); my @out = system_cmd_capture_noprint(@cmd); my $status = shift @out; if ($status == 0) { if (scalar(@out) > 0) { trace(join("\n> ", ("Command output:", @out)), "\n>End Command output"); } trace("Start of resource \"ora.asm\" Succeeded"); } elsif (scalar(grep(/CRS-0*5702/i, @out)) != 0) { # resource already up trace("The resource \"ora.asm\" was already running"); } elsif ($status != 0) { if (scalar(@out) > 0) { trace(join("\n> ", ("Command output:", @out)), "\n>End Command output"); } # if output contains ORA-15153 if (scalar(grep (/ORA-15153/i, @out)) > 0) { print_lines(@out); die(dieformat(551)); } if (!check_service("ora.asm", 10)) { print_lines(@out); trace("Start of resource \"ora.asm\" failed\n".join("\n", @out)); print_error(115, "ora.asm"); print_error(117); die(dieformat(247)); } else { trace("The resource \"ora.asm\" is already running"); } } } sub prepare_to_upgrade_siha { # Add the DB service sids from the ORA_ASMDBA group to the # ORA_ASMADMIN group if (($CFG->platform_family eq "windows") && (isOldVersion121010())) { s_migrateDBServiceSidsUpgrade(); } perform_siha_pre_upgrade(); } sub upgrade_olr_config { my $ckptName = "ROOTCRS_OLR"; if (isOLRConfiguredCkpt()) { return SUCCESS; } my $oraolr = $CFG->compOLR; $oraolr->upgradeCurrentNode($oraolr->UPDATE_OLRLOC); initial_cluster_validation(); if (FALSE == $oraolr->upgradeCurrentNode($oraolr->UPGRADE_OLR)) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(188)); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName("ROOTCRS_STACK"); trace("OLR initialization - successful"); return SUCCESS; } sub get_oldconfig_info { if (! $CFG->UPGRADE) { trace("Not a upgrade scenario"); return; } print_info(464); trace ("Retrieving older cluster configuration data"); my $oldCrsHome; my $ckptName = "ROOTCRS_OLDHOMEINFO"; my $globalCkptName = "ROOTCRS_OLDHOMEINFO"; my $ckptStatus; my $globalCkptStatus; my @oldCrsVer; my $verstring; my $oraocr = $CFG->compOCR; setConfiguredCRSHome(); $CFG->wipCkptName($ckptName); forceCheck(); my $isFirstNodeToUpgrade = FALSE; if (isFirstNodeToUpgrade()) { $isFirstNodeToUpgrade = TRUE; } if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); } else { writeCkpt($ckptName, CKPTSTART); } if (isCkptexist($globalCkptName, "-global")) { $globalCkptStatus = getCkptStatus($globalCkptName, "-global"); } else { writeGlobalCkpt($globalCkptName, CKPTSTART, "-transferfile"); } # Get Old CRS Home if ($globalCkptStatus eq CKPTSUC) { $oldCrsHome = getCkptPropertyValue($globalCkptName, "OLD_CRS_HOME", "-global"); $oldCrsHome = trim($oldCrsHome); trace("old crs home from ckpt property is $oldCrsHome"); } else { if ($isFirstNodeToUpgrade) { # Get old CRS home $oldCrsHome = getConfiguredCRSHome(); writeGlobalCkpt($globalCkptName, CKPTFAIL, "-transferfile") if (! $oldCrsHome); } } if (! $oldCrsHome) { trace("Failed to retrieve old Grid Infrastructure home location"); $CFG->wipCkptName("ROOTCRS_STACK"); die(dieformat(398)); } trace("Old CRS Home = $oldCrsHome"); $CFG->oldconfig('ORA_CRS_HOME', $oldCrsHome); $CFG->OLD_CRS_HOME($oldCrsHome); trace("Copying gpnp files to newer CRS home ..."); if (isFirstNodeToUpgrade()) { copy_gpnpglobalfiles($CFG->OLD_CRS_HOME, $CFG->ORA_CRS_HOME); } elsif (! isHomeShared()) { copy_gpnpglobalfiles($CFG->OLD_CRS_HOME, $CFG->ORA_CRS_HOME); } copy_gpnpnodefiles($CFG->OLD_CRS_HOME, $CFG->ORA_CRS_HOME); # Get old CRS version if ($globalCkptStatus eq CKPTSUC) { $verstring = getCkptPropertyValue($globalCkptName, "OLD_CRS_VERSION", "-global"); $verstring = trim($verstring); @oldCrsVer = split(/\./, $verstring); trace("old crs version from ckpt property is @oldCrsVer"); } elsif ($isFirstNodeToUpgrade) { # Get old CRS version (actually active version), # use the configured CRS home @oldCrsVer = get_crs_version(getConfiguredCRSHome()); trace("The active version of the Oracle Clustereware is '@oldCrsVer'"); if (! isValidVersion(join('.', @oldCrsVer))) { @oldCrsVer = split(/\./, getcrsrelver($CFG->OLD_CRS_HOME)); trace("The release vesion of the Oracle Clustereware is '@oldCrsVer'"); } } trace("Old CRS version is '@oldCrsVer'"); $CFG->oldconfig('ORA_CRS_VERSION', \@oldCrsVer); my $verinfo = join('.', @oldCrsVer); trace("Version Info retreived is : $verinfo"); #upgrade dry run when the old stack is running before upgrade proper if ($isFirstNodeToUpgrade && $CFG->platform_family ne "windows") { upgrade_dry_run(); } if (! isOldVersionLT121()) { my $asmCredsFile_old = catfile($CFG->OLD_CRS_HOME, 'gpnp', 'seed', 'asm', 'credentials.xml'); my $asmCredsFile_new = catfile($CFG->ORA_CRS_HOME, 'gpnp', 'seed', 'asm', 'credentials.xml'); if (-e $asmCredsFile_old) { trace("Copying ASM credentials file from the lower version GI home"); copy_file($asmCredsFile_old, $asmCredsFile_new); s_set_ownergroup($CFG->params('ORACLE_OWNER'), $CFG->params('ORA_DBA_GROUP'), $asmCredsFile_new); s_set_perms("0644", $asmCredsFile_new); } } verify_gpnp_dirs($CFG->ORA_CRS_HOME, $CFG->params('GPNPGCONFIGDIR'), $CFG->params('GPNPCONFIGDIR'), $CFG->HOST, $CFG->params('ORACLE_OWNER'), $CFG->params('ORA_DBA_GROUP')); trace("Updating the clusterwide gpnp stage profile with the latest node profile"); gpnp_update_config(); # Get current ASM mode trace("Get the current ASM mode"); my $asm_mode = undef; if (($globalCkptStatus eq CKPTSUC) && (isCkptPropertyExists($globalCkptName, "ASM_MODE", "-global"))) { $asm_mode = trim(getCkptPropertyValue($globalCkptName, "ASM_MODE", "-global")); trace("ASM mode from ckpt property: $asm_mode"); } elsif ($isFirstNodeToUpgrade) { # 12.1 onwards, ASM mode is available. if (isOldVersionLT121()) { trace("ASM mode: legacy"); $asm_mode = ASM_MODE_LEGACY; } else { trace("Try to read ASM mode from the node-specific profile"); my $rc = 0; ($rc, $asm_mode) = gpnp_get_asm_mode(get_peer_profile_file(TRUE)); if (0 != $rc) { trace("Unable to get ASM mode from Oracle Clusterware GPnP profile"); } trace("ASM mode: $asm_mode"); if ($asm_mode) { $asm_mode = trim($asm_mode); if ($asm_mode eq '') { trace("ASM mode is not present in the profile, " . "hence default to legacy"); $asm_mode = ASM_MODE_LEGACY; } } } writeCkptProperty($globalCkptName, "ASM_MODE", $asm_mode, "-global", "-transferfile"); } if (! $asm_mode) { trace("Undefined ASM mode: $asm_mode"); die(dieformat(509)); } else { trace("Store the current ASM mode: $asm_mode"); $CFG->oldconfig('ASM_MODE', $asm_mode); } # manual backup OCR for restore and store the backup info in OCR if ($isFirstNodeToUpgrade) { my $oraocr = $CFG->compOCR; trace("Old CRS home: ". $CFG->OLD_CRS_HOME); $oraocr->backupOcr4Restore($CFG->OLD_CRS_HOME); } # Get old HAIP ENABLED trace("Get old HAIP ENABLED value"); my $ev = -1; if (($globalCkptStatus eq CKPTSUC) && (isCkptPropertyExists($globalCkptName, "OLD_HAIP_ENABLED", "-global"))) { $ev = trim(getCkptPropertyValue($globalCkptName, "OLD_HAIP_ENABLED", "-global")); trace("Old HAIP ENABLED from ckpt property: $ev"); } elsif ($isFirstNodeToUpgrade) { $ev = get_HAIP_ENABLED_fromOld(); } trace("Old HAIP ENABLED is $ev"); if (-1 == $ev) { trace("Old HAIP_ENABLED value is invalid"); die(dieformat(624)); } else { $CFG->OLD_HAIP_ENABLED($ev); } # Get Current Node Config Role if ( ! isOldVersionLT121() ) { my $node_config_role; if ( ($ckptStatus eq CKPTSUC) && (isCkptPropertyExists($ckptName,'NODE_CONFIG_ROLE'))) { $node_config_role = trim(getCkptPropertyValue($ckptName, "NODE_CONFIG_ROLE")); } else { $node_config_role = getNodeConfigRole($CFG->HOST,$CFG->OLD_CRS_HOME); if (defined $node_config_role) { trace("Query the node role from OLR using crsctl, role is $node_config_role"); writeCkptProperty($CFG->wipCkptName, "NODE_CONFIG_ROLE", $node_config_role); } else { print_error(376, $CFG->HOST); } } $CFG->oldconfig('NODE_CONFIG_ROLE',$node_config_role); } if ($isFirstNodeToUpgrade) { # The old stack should be always up if (CKPTSUC eq $ckptStatus) { trace("Attempt to get old configuration info from the ckpt property"); } else { if (FAILED == check_OldCrsStack()) { trace("Make sure the older stack is completely down"); stopClusterware($oldCrsHome, "crs") || die(dieformat(349)); trace("The old stack isn't up, try to bring it up"); start_OldCrsStack(); } } } my $asmConf = -1; if ((CKPTSUC eq $globalCkptStatus) && (isCkptPropertyExists($globalCkptName, "ASM_CONFIGURED", "-global"))) { $asmConf = trim(getCkptPropertyValue($globalCkptName, "ASM_CONFIGURED", "-global")); trace("ASM_CONFIGURED from ckpt property: $asmConf"); } elsif ($isFirstNodeToUpgrade) { $asmConf = isASMConfigured(getConfiguredCRSHome()); trace("ASM is configured or not: $asmConf"); } if (-1 == $asmConf) { trace("Failed to retrieve the required information for ASM upgrade"); writeCkpt($ckptName, CKPTFAIL); writeGlobalCkpt($globalCkptName, CKPTFAIL, "-transferfile"); $CFG->wipCkptName("ROOTCRS_STACK"); exit 1; } else { trace(" asmconf =$asmConf"); $CFG->oldconfig('ASM_CONFIGURED', $asmConf); } # Retrieve the information about ACFS & volume resources when upgrade # from std ASM my $acfsreslist = "NULL"; if ($isFirstNodeToUpgrade && isLegacyASM()) { if ($ckptStatus eq CKPTSUC) { if (isCkptPropertyExists($ckptName, "ACFS_ADVM_RESOURCES_LIST")) { $acfsreslist = getCkptPropertyValue($ckptName, "ACFS_ADVM_RESOURCES_LIST"); $acfsreslist = trim($acfsreslist); trace("ACFS and volume resources list from ckpt property " . "is [$acfsreslist]"); } } else { $acfsreslist = ($CFG->compACFS)->getAllACFSandADVMRes($CFG->OLD_CRS_HOME); if (! $acfsreslist) { trace("No ACFS or volume resources found from the older stack"); $acfsreslist = "NULL"; } } trace (" ACFS & ADVM resources list = $acfsreslist"); } if ("NULL" ne $acfsreslist) { $CFG->ACFS_ADVM_RESOURCES_LIST($acfsreslist); } # Get the node list my $nodelist = $CFG->params('NODE_NAME_LIST'); trace("Node list retrieved from crsconfig param: $nodelist"); $CFG->oldconfig('NODENAME_LIST', $nodelist); trace (" old CrsHome = $oldCrsHome"); trace (" old CrsVer = @oldCrsVer"); trace (" node name list = $nodelist"); trace (" old HAIP ENABLED = $ev"); if ($ckptStatus ne CKPTSUC) { trace("Write old configuation data into the checkpoint file"); if ("NULL" ne $acfsreslist) { writeCkptProperty($CFG->wipCkptName, "ACFS_ADVM_RESOURCES_LIST", $acfsreslist); } } if ($globalCkptStatus ne CKPTSUC && $isFirstNodeToUpgrade) { trace("Write old configuation data into the global checkpoint file."); writeCkptProperty($globalCkptName, "OLD_CRS_HOME", $oldCrsHome, "-global"); writeCkptProperty($globalCkptName, "OLD_CRS_VERSION", $verinfo, "-global"); if (-1 != $asmConf) { writeCkptProperty($globalCkptName, "ASM_CONFIGURED", $asmConf, "-global"); } if (-1 != $ev) { writeCkptProperty($globalCkptName, "OLD_HAIP_ENABLED", $ev, "-global"); } } # get cvu resource values when old stack is running or else get from check point my $isCvuConfigured = 0; my $cvuCheckInterval = 0; my $isOldCvuRemoved = 0; # need to remove and add cvu resource because of AGENT_FILENAME attribute change # in 12.2 if ($isFirstNodeToUpgrade) { if (isOldVersionLT122 ()) { $isCvuConfigured = getCkptPropertyValue($globalCkptName, "IS_CVU_CONFIGURED", "-global"); if ("" eq $isCvuConfigured) { $isCvuConfigured = isCVUConfigured($oldCrsHome) ? 1 : 0; writeCkptProperty($globalCkptName, "IS_CVU_CONFIGURED", $isCvuConfigured, "-global", "-transferfile"); } $cvuCheckInterval = getCkptPropertyValue($globalCkptName, "CVU_CHECK_INTERVAL", "-global"); if (("" eq $cvuCheckInterval) && $isCvuConfigured) { $cvuCheckInterval = get_CVU_checkInterval($oldCrsHome); writeCkptProperty($globalCkptName, "CVU_CHECK_INTERVAL", $cvuCheckInterval, "-global", "-transferfile"); } $isOldCvuRemoved = getCkptPropertyValue($globalCkptName, "IS_OLD_CVU_REMOVED", "-global"); if (("" eq $isOldCvuRemoved) || (! $isOldCvuRemoved)) { trace ("removing old cvu"); set_flag_for_ocr_changed(); my $success = remove_CVU($CFG->OLD_CRS_HOME); if (!$success) { writeCkpt($ckptName, CKPTFAIL); writeGlobalCkpt($globalCkptName, CKPTFAIL, "-transferfile"); die(dieformat(563)); } writeCkptProperty($globalCkptName, "IS_OLD_CVU_REMOVED", 1, "-global"); writeCkptProperty($globalCkptName, "IS_CVU_CONFIGURED", 0, "-global", "-transferfile"); } trace ("cvu check interval = $cvuCheckInterval cvu config is $isCvuConfigured"); $CFG->oldconfig('IS_CVU_CONFIGURED', $isCvuConfigured); $CFG->oldconfig('CVU_CHECK_INTERVAL', $cvuCheckInterval); } elsif ($ckptStatus ne CKPTSUC) { set_flag_for_ocr_changed(); my $success = disable_CVU($CFG->OLD_CRS_HOME); if (!$success) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(564)); } } } # Attempt to retrieve the Cluster root certificate if ($CFG->isFirstNodeToUpgrade && ($ckptStatus ne CKPTSUC)) { trace("Attempting to extract existing cluster root certificate from " . "OCR-CREDSTORE..."); # cluster root certificate exists from 12.2.0.1.0 if (!isOldVersionLT122()) { # If old_version is 12.2.0.1, the cluster root certificate does not # exist for Application cluster if (isOldVersionLT12202() && isAppCluster()) { trace("Cluster root certificate does not exist for Application " . "cluster. New root certificate will be generated."); } else { my $wallet = catfile(get_configured_orabase($oldCrsHome), 'crsdata', tolower_host(), 'security', 'rootwallet'); # the certificate should exist; if it cannot be retrieved, die get_cluster_rootcert($wallet, $oldCrsHome); } } else { trace("Cluster root certificate does not exist. New root certificate " . "will be generated."); } } # Save attributes of OHASD resources to preserve them after upgrade if (($ckptStatus ne CKPTSUC) && ($CFG->compOHASD->saveOldAttrValues($oldCrsHome) == FAILED)) { die(dieformat(712)); } writeCkpt($ckptName, CKPTSUC) if ($ckptStatus ne CKPTSUC); if ($globalCkptStatus ne CKPTSUC) { writeGlobalCkpt($globalCkptName, CKPTSUC, "-transferfile"); } $CFG->wipCkptName("ROOTCRS_STACK"); print_info(465); return; } ####--------------------------------------------------------- #### Function for returning CLUSTER_GUID. # 1) First , checks for clusterware active version # NOTE: CFG->oldconfig('ORA_CRS_VERSION') must be set. # 2) If, version < 11.1.0.7 , returns -1. # 3) Else, get the clusterguid from "crsctl get css clusterguid". # ARGS: 1 # ARG1: ORA_CRS_HOME # @returns ID fetched from crsctl or -1 in case of error sub get_clusterguid { my $home = $_[0]; my $id = -1; ## If here , must be 11.1.0.7 and higher my @OLD_CRS_VERSION = @{$CFG->oldconfig('ORA_CRS_VERSION')}; my $old_crs_ver = join('.', @OLD_CRS_VERSION); if (-1 == versionComparison($old_crs_ver, "11.1.0.7.0")) { trace("Skipping clusterguid fetch for ".join('.',@OLD_CRS_VERSION)); return -1; } trace("Fetching clusterguid from ".join('.',@OLD_CRS_VERSION)); my $cmd; if (! defined $home) { $cmd = crs_exec_path('crsctl'); } else { $cmd = catfile( $home, 'bin', 'crsctl' ); } # run "crsctl get css clusterguid" my @out = system_cmd_capture(($cmd, "get", "css", "clusterguid")); my $rc = shift @out; # if succeeded, get the guid, output must be a single line if ($rc == 0) { my $outid = $out[0]; $id = trim($outid); trace( "Got CSS GUID: $id (".join(' ',@out).")" ); } else { trace ("Retrieval of CSS GUID failed (rc=$rc), ". "with the message:\n".join("\n", @out)."\n"); } if ($id == -1) { trace("get_clusterguid : Failed to get clusterguid"); } return $id; } =head2 get_oifcfg_iflist Gets "oifcfg iflist" networks interface info, e.g. ("eth0 10.0.0.0 PRIVATE 255.255.252.0", "eth1 140.87.4.0 UNKNOWN 255.255.252.0") Note that adapter name (e.g. eth1) can be quoted and contain spaces on some platforms, and ip net addr can be ipv6. =head3 Parameters string with oifcfg home location. If undef, then current home is used. =head3 Returns =head4 returns a list of strings-net intf defs =head4 result code (0 for success) as a first member of array. =cut sub get_oifcfg_iflist { return get_oifcfg_info(($_[0], 'iflist','-p','-n')); } =head2 get_oifcfg_getif Gets "oifcfg getif" networks interface info, e.g. ("eth0 10.0.0.0 global public", "eth1 140.87.4.0 global cluster_interconnect") Note that adapter name (e.g. eth1) can be quoted and contain spaces on some platforms, and ip net addr can be ipv6. =head3 Parameters string with oifcfg home location. If undef, then current home is used. =head3 Returns =head4 returns a list of strings-net intf defs =head4 result code (0 for success) as a first member of array. =cut sub get_oifcfg_getif { return get_oifcfg_info(($_[0], 'getif')); } =head2 get_ocr_privatenames_info Gets OCR information about configured private nodenames in form indentical to "olsnodes -p" output (nodename private_names...). This is used to replace olsnodes-p where not available, e.g. on 10.1. OCR access performed with ORA_CRS_HOME ocrdump utility by dumping SYSTEM.css keys. If all fails, will try "olsnodes" (node names only) in old home as a last resort. =head3 Parameters string with olsnodes home location. If undef, then current home is used. =head3 Returns =head4 returns a list of strings with node names. =head4 result code (0 for success) as a first member of array. =cut sub get_ocr_privatenames_info { my $home = $_[0]; my @nodes = (); my $cmd; if (! defined $home) { $cmd = crs_exec_path('ocrdump'); } else { $cmd = catfile( $home, 'bin', 'ocrdump' ); } # run "ocrdump -stdout -keyname SYSTEM.css" my @args = ($cmd, '-stdout', '-keyname', 'SYSTEM.css'); my @out = system_cmd_capture(@args); my $rc = shift @out; if ($CFG->DEBUG) { trace "---SYSTEM.css OCR dump:\n". join(' ',@args)."\nout: \n".join("\n",@out)."\n"; } # read-in dumped css keys if (0 == $rc) { my @nodnames = grep(/^\[SYSTEM\.css\.node_names\.[^\]\.]+\]$/, @out); my @pvtnames = grep(/^\[SYSTEM\.css\.privatenames\.[^\]\.]+\]$/, @out); if (!(@nodnames) || !(@pvtnames)) { trace "Warning: OCR has no css public or private node names. "; } if ($CFG->DEBUG) { trace "---OCR node_names: ". join(' ',@nodnames). "\n---OCR pvt_names: ". join(' ',@pvtnames); } # to keep it simple, we do not do any matching by nodenum between # node_names and privatenames, since they are not used together anyways; # assuming same order, same number. foreach (0..$#nodnames) { my $curidx = $_; my $nodname = $nodnames[$_]; if (defined $nodname) { $nodname =~ m/^.+\.([^\]\.]+)\]$/; # take last key - nodename $nodname = $1; } # normally both arrays will be paired my $pvtname = $pvtnames[$_]; if (defined $pvtname) { $pvtname =~ m/^.+\.([^\]\.]+)\]$/; # take last key - nodename $pvtname = $1; } if (defined $nodname) { push @nodes, "$nodname $pvtname"; } trace ("ocr node parsed: -$nodname-$pvtname-="); } if (scalar(@nodes) == 0) { trace("Failed to get a list of CSS nodes from OCR. ". "Setup way not work properly."); print_error(171); # return at least a list of nodes (no -p), so setup can propagate # properly - run in old home. return get_olsnodes_info($CFG->OLD_CRS_HOME); } } else { push @nodes, "".join(' ', @args)." failed."; } return ($rc, @nodes); } =head2 parse_netinfo Parse oifcfg-style netinfo (iflist/getif) into array of refs to array for each interface, containing interface definition elements. See oifcfg_intf_parse. =head3 Parameters An array reference of strings with oifcfg output. =head3 Returns =head4 returns a resulting array of parsed interfaces, each represented as an array containing string components of interface definition (interface name (unquoted), masked addr, scope (global/node), type (public,cluter_interconnect), mask). =head4 result code (0 for success) as a first member of array. =cut sub parse_netinfo { my $netoutref = $_[0]; #ref my @intfs = @{$netoutref}; my @netinfo = (); my $rc = 0; foreach (0..$#intfs) { my $idef = $intfs[$_]; my $ada; my $net; my $nod; my $typ; my $msk; my @net = oifcfg_intf_parse( $idef ); ($ada, $net, $nod, $typ, $msk ) = @net; if ((defined $typ) && (defined $net) && (defined $ada)) { push @netinfo, \@net; } } return ($rc, @netinfo); } =head2 parse_olsnodesp_netinfo Parse olsnodes-style netinfo into array of refs to array for each node info. Each array contains public and private node name. =head3 Parameters An array reference of strings with olsnodes output. =head3 Returns =head4 returns an array of parsed olsnodes node info arrays (public/private node name). =head4 result code (0 for success) as a first member of array. =cut sub parse_olsnodesp_netinfo { my $netoutref = $_[0]; #ref my @intfs = @{$netoutref}; my @netinfo = (); my $rc = 0; foreach (0..$#intfs) { my $idef = $intfs[$_]; my $n; my $host; my $pvthost; if ($CFG->DEBUG) { trace ("intf: $idef"); } # olsnodes -p will give output in form " " lines. $idef =~ s/^\s+|\s+$//g; $idef =~ s/\s+/ /g; $n = rindex( $idef, ' ' ); $host = substr( $idef, 0, $n ); $pvthost = substr( $idef, $n+1 ); my @net = ($host, $pvthost); if (defined $host) { push @netinfo, \@net; } if ($CFG->DEBUG) { trace ("node parsed: -$host-$pvthost-="); } } return ($rc, @netinfo); } =head2 match_node_netintfs Create a table of oifcfg-style cluster-interconnect interfaces based on oldnodes -p private node names, resolved and matched against available node interfaces. =head3 Parameters =head4 an array reference of parsed olsnodes-p info (refs to parsed lines). =head4 an array reference of parsed oifcfg-iflist info (refs to parsed lines) =head3 Returns =head4 returns an array of parsed oifcfg-iflist style info (array of refs) for cluster-interconnect interfaces inferred from olsnodes-p output. =head4 result code (0 for success) as a first member of array. =cut sub match_node_netintfs { my $olsref = $_[0]; #ref my $oiflstref = $_[1]; #ref my @olshs = @{$olsref}; # array of parsed olsnodes host/pvthost arrays my @intfs = @{$oiflstref}; # array of parsed oifcfg iflist arrays my @netinfo = (); my $rc = 0; trace "Processing ".scalar(@olshs)." olsnodes:"; foreach (0..$#olshs) { my $olsintfref = $olshs[$_]; my $node; my $pvtnode; my $iaddr; my $saddr; ($node, $pvtnode) = @{$olsintfref}; trace " node $_ pub:$node pvt:$pvtnode="; # [ resolve private node into addr if (defined $pvtnode) { my $name; my $aliases; my $addrtype; my $length; my @addrs; ($name, $aliases, $addrtype, $length, @addrs) = gethostbyname $pvtnode or die(dieformat(253, $pvtnode, $!)); (($addrtype == AF_INET) && (scalar(@addrs) > 0)) or next; ($length == 4) or print_error(34); # 16 trace " $pvtnode addrs: "; for my $iiaddr (@addrs) { $saddr = undef; $iaddr = undef; if (defined $iiaddr) { $saddr = inet_ntoa( $iiaddr ); trace " $saddr "; # toberevised: +ipv6 - inet_pton $iaddr = ipv4_atol($saddr); } if (defined $saddr) { # Now loop through the iflist interfaces and see if # node private addr matched for known adapter foreach (0..$#intfs) { my $intfref = $intfs[$_]; my $ada; my $net; my $nod; my $typ; my $msk; ($ada, $net, $nod, $typ, $msk ) = @{$intfref}; if ((defined $msk) && (defined $net) && (defined $ada)) { # toberevised: +ipv6 - inet_pton my $imask = ipv4_atol($msk); my $inet = ipv4_atol($net); my $match = FALSE; $match = TRUE if (($imask & $iaddr) == $inet); my $unique = TRUE; if ($match) { foreach (0..$#netinfo) { my $intfref1 = $netinfo[$_]; my $ada1 = @{$intfref1}[0]; if ($ada1 eq $ada) { $unique = FALSE; last; } } if ($unique) { # make a new cluster_interconnect intf my @intf = ($ada, $net, 'global', 'cluster_interconnect', $msk ); push @netinfo, \@intf; } } trace(" matching olsnodes/iflist ". "(net $net == $saddr & $msk) match=$match ". "unique=$unique"); } } } } } # ] } return ($rc, @netinfo); } =head2 match_getif_netintfs Create a consolidated table of all used oifcfg-style interfaces based on oifcfg getif info, ammended with resolved oldnodes -p info, if any. Resulting netinfo will be used as a NETWORKS info in the gpnp profile. =head3 Parameters =head4 an array reference of parsed oifcfg-getif info (refs to parsed lines). =head4 an array reference of parsed oifcfg-olsnodes info from match_node_netintfs (refs to parsed lines) =head3 Returns =head4 returns a merged array of parsed oifcfg-getif style info (array of refs) for available network interfaces. =head4 result code (0 for success) as a first member of array. =cut sub match_getif_netintfs { my $getifref = $_[0]; #ref my $olsifref = $_[1]; #ref my @getifs = @{$getifref}; # array of parsed oifcfg getifs arrays my @olsifs = @{$olsifref}; # array of parsed olsnodes matched to iflist my @netinfo = (); my $rc = 0; my $ada; my $net; my $nod; my $typ; my $msk; my $getif_pvt_ifs = 0; my $getif_pub_ifs = 0; # copy getif array into resulting netinfo as base, to be ammended with # olsnodes info foreach (0..$#getifs) { my $gifref = $getifs[$_]; ($ada, $net, $nod, $typ, $msk ) = @{$gifref}; if (defined $ada) { my @netif = ($ada, $net, $nod, $typ, $msk ); push @netinfo, \@netif; # add unconditionally # count if types if ($typ =~ m/cluster_interconnect/i) { $getif_pvt_ifs++; } if ($typ =~ m/public/i) { $getif_pub_ifs++; } } } # If getif has no interconnects, try to derive them from # olsnodes/ocr-private-name info: # loop through olsnodes interfaces and see if there is something not # yet present in results # # Note: ALL olsifs are cluster_interconnects # if ( $getif_pvt_ifs == 0 ) { foreach (0..$#olsifs) { my $olsifref = $olsifs[$_]; ($ada, $net, $nod, $typ, $msk ) = @{$olsifref}; my $match = FALSE; my $typealt = FALSE; if (defined $ada) { foreach (0..$#netinfo) { my $netifref = $netinfo[$_]; my $ada_if; my $net_if; my $nod_if; my $typ_if; my $msk_if; ($ada_if, $net_if, $nod_if, $typ_if, $msk_if ) = @{$netifref}; # 0 1 2 3 4 # if a different interface with the same subnet found in results # do not add it, just update the type. # (Loose match on subnet, not adapter name if ($ada eq $ada_if)) # if ($net eq $net_if) { $match = TRUE; # make sure olsnodes type is included in the results type # - If there are multiple public types defined, override # matching interfaces to be cluster_interconnects # - If there is only single public, make it dual-purpose # if ($typ_if !~ m/$typ/i) { $typealt = TRUE; if ($getif_pub_ifs > 1) { $getif_pub_ifs--; ${$netifref}[3] = $typ; # replace } else { ${$netifref}[3] = $typ_if .= ",$typ"; # append } } last; } } # if olsnodes interface is not int the results yet, add a copy if (! $match) { my @olsif = @{$olsifref}; push @netinfo, \@olsif; } trace(" matching olsnodes/getif $ada-$net match=$match ". "typealt=$typealt "); } } # for olsifs } trace "---resulting upgrade iflist:"; foreach (0..$#netinfo) { my $netifref = $netinfo[$_]; ($ada, $net, $nod, $typ, $msk ) = @{$netifref}; trace ("intf $_: -$ada-$net-$nod-$typ-$msk-"); } trace "---"; return ($rc, @netinfo); } =head2 get_upgrade_node_list Create a comma-seaparated list of cluster nodes based on parsed oldnodes info. =head3 Parameters =head4 an array reference of parsed olsnodes-p info (refs to parsed lines). =head3 Returns =head4 returns a comma-separated list of cluster nodes. =cut sub get_upgrade_node_list { my $olsref = $_[0]; #ref my @olshs = @{$olsref}; # array of parsed olsnodes host/pvthost arrays my $s_nodes_list = ""; foreach (0..$#olshs) { my $olsintfref = $olshs[$_]; my ($node, $pvtnode) = @{$olsintfref}; if (defined $node) { if (! ($s_nodes_list eq "")) { $s_nodes_list .= ","; } $s_nodes_list .= $node; } } if ($CFG->DEBUG) { trace "Cluster node list, per olsnodes: \"$s_nodes_list\""; } return $s_nodes_list; } =head2 get_ocr_getif_info Gets OCR information about configured interfaces in form indentical to "oifcfg getif" output, e.g. line "Local Area Connection 4 140.87.136.0 global cluster_interconnect,public" see get_oifcfg_getif() This is used to replace "oifcfg getif" when stack is not running during rolling upgrade from older pcw versions. OCR access performed with ORA_CRS_HOME ocrdump utility by dumping SYSTEM.css.interfaces.global keys. =head3 Parameters string with ocrdump home location. If undef, then current home is used. =head3 Returns =head4 returns a list of strings with "oifcfg getif"-like output. =head4 result code (0 for success) as a first member of array. =cut sub get_ocr_getif_info { my @intfs = (); my $home = $_[0]; my $cmd; if (! defined $home) { $cmd = crs_exec_path('ocrdump'); } else { $cmd = catfile( $home, 'bin', 'ocrdump' ); } # run "ocrdump -stdout -keyname SYSTEM.css.interfaces.global" my @args = ($cmd, '-stdout', '-keyname', 'SYSTEM.css.interfaces.global'); my @out = system_cmd_capture(@args); my $rc = shift @out; if ($CFG->DEBUG) { trace "---SYSTEM.css.interfaces.global OCR dump:\n". join(' ',@args)."\nout: \n".join("\n",@out)."\n"; } # read-in dumped css keys if (0 == $rc) { foreach (0..$#out) { my $curidx = $_; my $key; my $itf; my $ada; my $net; my $typ; # e.g. # [SYSTEM.css.interfaces.global.eth0.192|d168|d1|d0.1] # ORATEXT : cluster_interconnect,public $key = $out[$_]; $ada = $net = $typ = $itf = undef; if ($key =~ m/^\[SYSTEM\.css\.interfaces\.global\.([^\]^\.]+)\.([^\]^\.]+)\.1\]$/) { # take last key - nodename $ada = $1; $net = $2; if (defined $ada && defined $net) { $net =~ s/\|d/\./g; # for ipv4, make ip notation $key = $out[$_+1]; if ( $key =~ m/^ORATEXT : (.+)$/) { # take type classification for current network $typ = $1; $itf = "$ada $net global $typ"; } if ($CFG->DEBUG) { trace ("ocr intf parsed: [$curidx] -$ada-$net-$typ-="); } } } if (defined $itf) { push @intfs, $itf; } } trace ("---ocr iflist:"); trace_lines(@intfs); trace ("---"); if (scalar(@intfs) == 0) { $rc = 2; trace("Failed to get a list of interfaces from OCR. ". "Setup may not work properly."); print_error(172); } } else { push @intfs, "".join(' ', @args)." failed."; } return ($rc, @intfs); } sub isNodeappsExists #------------------------------------------------------------------------------- # Function: Check if nodeapps exists # Args : none # Returns : TRUE if exists # FALSE if not exists #------------------------------------------------------------------------------- { my @crs_nodevip_list_old = get_OldVipInfoFromOCRDump(); if (scalar(@crs_nodevip_list_old) == 0) { return FALSE; } return TRUE; } # Stops the old running crs stack. sub stop_OldCrsStack { my $OLD_CRS_HOME = $CFG->OLD_CRS_HOME; my $status = s_stop_OldCrsStack($OLD_CRS_HOME); if (0 == $status) { trace ("Old CRS stack stopped successfully"); } else { trace ("Unable to stop Old CRS stack"); die; } sleep(60); } # Start the old CRS stack. sub start_OldCrsStack { my $old_crshome = $CFG->OLD_CRS_HOME; my $crsctl = catfile ($old_crshome, 'bin', 'crsctl'); my $rc; my @output; my $retries = 25; my $is_up = FALSE; if(! isOldVersionLT121()) { $rc = startFullStack($old_crshome); if($rc != SUCCESS) { die(dieformat(117)); } trace("old Grid Infrastructure stack started successfully"); return; } @output = system_cmd_capture($crsctl, 'start', 'crs'); $rc = shift (@output); if (0 != $rc) { print_lines(@output); trace("\"$crsctl start crs\" failed with status $rc"); die(dieformat(117)); } while ($retries) { if (check_OldCrsStack()) { $is_up = TRUE; last; } trace ("Waiting for old Grid Infrastructure stack to start"); sleep (5); $retries--; } if ($is_up) { trace ("old Grid Infrastructure stack started successfully"); } else { die(dieformat(120)); } } sub check_OldCrsStack #------------------------------------------------------------------------------- # Function: Check if the stack is up from old CrsHome # Args : none #------------------------------------------------------------------------------- { trace("check old crs stack"); my $old_crshome = $CFG->OLD_CRS_HOME; my $crsctl = catfile ($old_crshome, 'bin', 'crsctl'); my $status = FAILED; my @output = system_cmd_capture($crsctl, 'check', 'crs'); my $rc = shift @output; if ((0 == $rc) && (scalar(grep(/4537/, @output)) > 0) && (scalar(grep(/4529/, @output)) > 0) && (scalar(grep(/4533/, @output)) > 0)) { $status = SUCCESS; } if ($status == SUCCESS) { trace ("Earlier version Oracle Clusterware is running"); } else { trace ("Earlier version Oracle Clusterware is not running"); } return $status; } #update ons.config - Bug 8424681 sub update_ons_config { my @nodelist = getONSnodelist(); my $node; my $host = $CFG->HOST; my $str = "nodes="; my $ONSCONFFILE = catfile($CFG->params('ORACLE_HOME'), 'opmn' , 'conf', 'ons.config'); foreach $node (@nodelist) { my($onslocport, $onsremport) = get_ons_port($node); trace ("ons remoteport for $node is $onsremport"); if ($node ne $nodelist[-1]) { $str = $str . "$node:$onsremport" . ","; } else { $str = $str . "$node:$onsremport"; } } trace("ons nodes string is $str"); my($onslocport, $onsremport) = get_ons_port($host); trace("ons conf file is $ONSCONFFILE"); open(FONS, ">>$ONSCONFFILE") or print_error(185, $ONSCONFFILE, $!); print FONS "localport=$onslocport\n"; print FONS "remoteport=$onsremport\n"; print FONS "$str\n"; close FONS; } sub getONSnodelist #--------------------------------------------------------------------- # Function: Get the list of nodes where ONS is configured. The nodes could # be outside of the cluster nodes as well(Bug 8563905). #--------------------------------------------------------------------- { my $cmd; my @nodelist; my $ORA_CRS_HOME = $CFG->ORA_CRS_HOME; my ($ONSCONFFILE, $line, $e, $e1, $e2, $e3, $e4); my ($home, $onsnodes, $node, $port, $Name, $list); my @tmparr; my @nodelistocr; my @nodelistons; my %union = (); $home = $CFG->OLD_CRS_HOME; $ONSCONFFILE = catfile( $home, 'opmn' , 'conf', 'ons.config'); trace("ons conf file is $ONSCONFFILE"); open(FONS, $ONSCONFFILE) or trace("Could not open \"$ONSCONFFILE\": $!"); while() { if(/^nodes\b/i) { $onsnodes=$_; } } close (FONS); ($Name, $list) = split(/=/, $onsnodes); @tmparr = split(/,/, $list); foreach my $elem (@tmparr) { ($node, $port) = split(/:/, $elem); push @nodelistons, $node; } trace("ons node list from ons.config is @nodelistons\n"); #Get the ons nodelist from OCR $cmd = catfile( $ORA_CRS_HOME, 'bin', 'ocrdump' ); my $ocrkey = "DATABASE.ONS_HOSTS"; my @args = ($cmd, '-stdout', '-keyname', $ocrkey); my @out = system_cmd_capture(@args); my $rc = shift @out; foreach $line (@out) { if($line =~ m/PORT\]/i) { ($e1, $e2, $e3, $e4) = split(/\./, $line); trace("ONS is configured on node $e3. Adding to onsnodelist"); $e3 =~ s/!/./g; push @nodelistocr, $e3; } } trace("ons nodelist from OCR is @nodelistocr\n"); foreach $e (@nodelistons) {$union{$e} = 1 } foreach $e (@nodelistocr) {$union{$e} = 1 } @nodelist = keys %union; trace("union of on nodelist from ons.config and OCR is @nodelist\n"); return @nodelist; } # gets the VIp information from the OCR. CRS stack needs to be up # before calling this sub routine. sub get_OldVipInfo { my @CRS_NODEVIP_LIST; my $vip_index = 0; my $OLD_CRS_HOME = $CFG->OLD_CRS_HOME; my $nodelist = $CFG->oldconfig('NODENAME_LIST'); my @SNODES = split (/,/, $nodelist); my ($vip_name, $ip, $old_netmask, $intif); my ($new_netmask, $vip, $rc, @out); my $run_as_owner = FALSE; # if version is 10.1, use dbhome. Otherwise, use OLD_CRS_HOME. my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')}; foreach my $nodename (@SNODES) { $rc = srvctl_capture($run_as_owner, \@out, "config nodeapps -n $nodename -S 1", $OLD_CRS_HOME); if (($rc != 0) && ($rc != 2)) { print_lines(@out); die(dieformat(289)); } my @buffer = grep(/ip=/, @out); if (scalar(@buffer) > 0) { trace ("buffer=$buffer[0]"); my @list = split(/ +/, $buffer[0]); foreach (@list) { if ($_ =~ /name=/) { $vip_name = parseText ($_); } elsif ($_ =~ /ip=/) { $ip = parseText ($_); } elsif ($_ =~ /netmask=/) { $old_netmask = parseText ($_); } elsif ($_ =~ /interfaces=/) { $intif = parseText ($_); } } } else { trace("Could not find IP details from running the command 'srvctl config nodeapps -n $nodename -S 1"); } chomp $vip_name; chomp $ip; chomp $old_netmask; chomp $intif; $intif =~ s/\:/\|/g; trace("Older VIP IP address (vip_name) = $vip_name"); trace("Older VIP IP name(ip) = $ip"); trace("Older VIP IP netmask (old_netmask) = $old_netmask"); trace("Older VIP IP interface (intif) =$intif"); # use vip_name if it exists, otherwise use ip if (! $vip_name) { $vip = $ip; } else { $vip = $vip_name; } if (validateNetmask($old_netmask, $intif, \$new_netmask)) { if ($intif) { $CRS_NODEVIP_LIST[$vip_index] = "$vip/$new_netmask/$intif"; } else { $CRS_NODEVIP_LIST[$vip_index] = "$vip/$new_netmask"; } } else { if ($intif) { $CRS_NODEVIP_LIST[$vip_index] = "$vip/$new_netmask/$intif"; } else { $CRS_NODEVIP_LIST[$vip_index] = "$vip/$old_netmask"; } } trace ("vip on $nodename = $CRS_NODEVIP_LIST[$vip_index]"); $vip_index++; } sub parseText { # extract netmask and interface my $line = $_[0]; $line =~ s/{//g; $line =~ s/}//g; my ($dummy, $text) = split (/=/, $line); chomp $text; return $text; } $ENV{'ORACLE_HOME'} = $CFG->params('ORACLE_HOME'); return @CRS_NODEVIP_LIST; } sub get_OldVipInfoFromOCRDump #------------------------------------------------------------------------------- # Function: Get old VIP info from ocrdump # Args : none # Returns : @vip_list #------------------------------------------------------------------------------- { trace("Getting old VIP details from ocrdump"); my $ocrdump = catfile ($CFG->params('ORACLE_HOME'), 'bin', 'ocrdump'); my $nodelist = $CFG->oldconfig('NODENAME_LIST'); my @nodes = split (/,/, $nodelist); my $ix = 0; my @vip_list; foreach my $nodename (@nodes) { $nodename = lc($nodename); # get IP from DATABASE.NODEAPPS.$nodename.VIP.IP if ($CFG->platform_family eq "windows") { open (OCRDUMP, "$ocrdump -stdout -keyname " . "DATABASE.NODEAPPS.$nodename.VIP.IP |"); } else { open (OCRDUMP, "$ocrdump -stdout -keyname " . "'DATABASE.NODEAPPS.$nodename.VIP.IP' |"); } my @output = ; close (OCRDUMP); trace("Dump from OCR: @output"); my @txt = grep (/ORATEXT/, @output); my ($key, $ip) = split (/: /, $txt[0]); chomp($ip); trace("IP address (found from dump) = $ip"); # get NETMASK from DATABASE.NODEAPPS.$nodename.VIP.NETMASK if ($CFG->platform_family eq "windows") { open (OCRDUMP, "$ocrdump -stdout -keyname " . "DATABASE.NODEAPPS.$nodename.VIP.NETMASK |"); } else { open (OCRDUMP, "$ocrdump -stdout -keyname " . "'DATABASE.NODEAPPS.$nodename.VIP.NETMASK' |"); } @output = ; close (OCRDUMP); trace("Dump from OCR: @output"); @txt = grep (/ORATEXT/, @output); my $old_netmask; ($key, $old_netmask) = split (/: /, $txt[0]); chomp($old_netmask); trace("IP netmask (found from dump) = $old_netmask"); # get network interface name if ($CFG->platform_family eq "windows") { open (OCRDUMP, "$ocrdump -stdout -keyname " . "CRS.CUR.ora!$nodename!vip.USR_ORA_IF |"); } else { open (OCRDUMP, "$ocrdump -stdout -keyname " . "'CRS.CUR.ora!$nodename!vip.USR_ORA_IF' |"); } @output = ; close (OCRDUMP); trace("Dump from OCR: @output"); @txt = grep (/ORATEXT/, @output); my $intif; ($key, $intif) = split (/: /, $txt[0]); my $new_netmask; chomp($intif); trace("Network interface (found from dump)=$intif"); if ($ip ne "" && $old_netmask ne "") { if (validateNetmask($old_netmask, $intif, \$new_netmask)) { if ($intif) { $vip_list[$ix] = "$ip/$new_netmask/$intif"; } else { $vip_list[$ix] = "$ip/$new_netmask"; } } else { if ($intif) { $vip_list[$ix] = "$ip/$new_netmask/$intif"; } else { $vip_list[$ix] = "$ip/$old_netmask"; } } $ix++; } } if ($CFG->DEBUG) { trace("vip_list = @vip_list"); } return @vip_list; } sub validateNetmask #------------------------------------------------------------------------------- # Function: Validate netmask # Args : [0] - old netmask # [1] - network interface # [1] - new netmask # Returns : TRUE if success # FALSE if failed # new netmask #------------------------------------------------------------------------------- { my $old_netmask = $_[0]; my $netif = $_[1]; my $new_netmask_ref = $_[2]; my $oifcfg = catfile($CFG->OLD_CRS_HOME, 'bin', 'oifcfg'); my $success = TRUE; $$new_netmask_ref = $old_netmask; # if version is 10.1, use new GRID home for oifcfg # 10.1 oifcfg does not support -n option. my @old_version = @{$CFG->oldconfig('ORA_CRS_VERSION')}; if ($old_version[0] eq "10" && $old_version[1] eq "1") { $oifcfg = catfile($CFG->ORA_CRS_HOME, 'bin', 'oifcfg'); } open OPUT, "$oifcfg iflist -p -n |"; if ($netif) { #not null my @output = grep { /\b$netif\b/i } (); trace("OIFCFG_IFLIST output : @output"); if (scalar(@output) == 0) { #not found trace("Unable to find netmask for network interface=$netif"); $success = FALSE; } else { if ($CFG->DEBUG) { trace("Processing OIFCFG_IFLIST output: $output[0]"); } my @netmask_list = split (/ +/, $output[0]); if (scalar(@netmask_list) > 2) { $$new_netmask_ref = $netmask_list[-1]; #last entry chomp $$new_netmask_ref; if ($old_netmask ne $$new_netmask_ref) { trace("old_netmask=$old_netmask does NOT match " . "new_netmask=$$new_netmask_ref"); $success = FALSE; } } else { $success = FALSE; } } } else { my @output = grep { /\b$old_netmask\b/i } (); trace("Processing OIFCFG IFLIST output: @output"); if (scalar(@output) == 0) { #not found trace("Unable to find netmask=$old_netmask"); $success = FALSE; } } close OPUT; return $success; } #get CRS sandbox location (for upgrade dry run) sub get_crs_sandbox_loc { my $sbloc = catfile($CFG->params('ORACLE_BASE'), 'crsdata', tolower_host(), 'sandbox'); return $sbloc; } sub sb_save_env_vars { foreach my $envVar (@SANDBOX_ENV_VARS_SET) { if (defined $ENV{$envVar}) { my $envVarSave = $envVar . "_SAVE"; $ENV{$envVarSave} = $ENV{$envVar}; } } } sub sb_restore_env_vars { foreach my $envVar (@SANDBOX_ENV_VARS_SET) { my $envVarSave = $envVar . "_SAVE"; if (defined $ENV{$envVarSave}) { $ENV{$envVar} = $ENV{$envVarSave}; } else { delete $ENV{$envVar}; } } delete $ENV{'CRS_SANDBOX_MODE'}; delete $ENV{'CRS_SANDBOX_LOC'}; } sub sbdietrap { trace("----Sandbox Warning-------"); backtrace(0); trace("----Sandbox Warning-------"); } sub create_crs_sandbox_dir() { my $sbloc = get_crs_sandbox_loc(); if (-e $sbloc) { rmtree($sbloc); rmdir($sbloc); } mkdir($sbloc); } ############################################################################## #Sub : cleanup_previous_sandbox #Args : None #Desc : Stops any running sandbox mode Ohasd/Crsd ############################################################################## sub cleanup_previous_sandbox() { my $crshome = $CFG->ORA_CRS_HOME; my $crsctl = catfile($crshome, 'bin', 'crsctl'); trace("enter cleanup_previous_sandbox"); #enter Sandbox mode $ENV{'CRS_SANDBOX_MODE'} = 'TRUE'; my $cmd = "$crsctl sandbox clean"; my @output = system_cmd_capture("$cmd"); my $status = shift @output; trace ("crsctl sandbox clean output = @output, status = $status"); delete $ENV{'CRS_SANDBOX_MODE'}; trace("Checking for stale sandbox processes"); #26895782:double-check and kill any lingering processes my $sbloc = get_crs_sandbox_loc(); my @pidFiles = glob(catfile($sbloc, "*.pid")); foreach my $pidFile (@pidFiles) { trace("Peek at the pid file <$pidFile>"); my @lines = read_file($pidFile); my $pid = trim($lines[0]); trace("PID in <$pidFile> is: $pid"); if ($pid =~ /^[0-9]+$/) { if ($CFG->platform_family ne "windows") { my $procCheck = s_verify_PID($pid, $pidFile);#ensure sandbox ohasd/crsd next if (! $procCheck); } } trace("Killing stale process pid = $pid"); kill(9, $pid); } return SUCCESS; } sub upgrade_dry_run { my $crshome = $CFG->ORA_CRS_HOME; my $oraocr = $CFG->compOCR; my $oraolr = $CFG->compOLR; my $crsctl = catfile($crshome, 'bin', 'crsctl'); my $status; my @output; my $cmd; my $orasrvm = $CFG->compSRVM; my $rc = FAILED; my $upgradeModelStatus; trace("Starting upgrade dry run"); if (lc($CFG->CHECKCRSENTITIES) eq 'no') { trace("Skipping upgrade dry run"); return SUCCESS; } print_info(692); cleanup_previous_sandbox(); #start old version stack if not running if (!check_OldCrsStack()) { trace("Old CRS not running, starting CRS to check upgrade status"); my $ckptName = "ROOTCRS_UPGPRECHECKS"; my $ckptStatus = CKPTSTART; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); } if ($ckptStatus eq CKPTSUC) { print_info(693); return SUCCESS; } start_OldCrsStack(); } sb_save_env_vars(); create_crs_sandbox_dir(); # Take the backup of old OCR if ($oraocr->setup_sandbox_ocr($oraocr->SANDBOX_BACKUP_OCR) == FAILED) { goto setErrorExit; } # Setup sandbox OLR if ($oraolr->setup_sandbox_olr() == FAILED) { goto setErrorExit; } # Setup sandbox OCR if ($oraocr->setup_sandbox_ocr($oraocr->SANDBOX_CONFIGURE_OCR) == FAILED) { goto setErrorExit; } #enter sandbox mode if (!is_dev_env()) { $ENV{'T_HAS_WORK'} = get_crs_sandbox_loc(); } $ENV{'CRS_SANDBOX_MODE'} = 'TRUE'; $ENV{'CRS_SANDBOX_LOC'} = get_crs_sandbox_loc(); $ENV{'_SCLS_DEVELOPMENT'} = 'TRUE'; $ENV{'HAS_DEVELOPMENT_ENVIRONMENT'} = 'TRUE'; if (!defined($ENV{'CSS_CLUSTERNAME'})) { $ENV{'CSS_CLUSTERNAME'} = "TEST67718"; } if (start_sandbox_ohasd($crshome) == FAILED) { goto setErrorExit; } if (start_sandbox_crsd($crshome) == FAILED) { goto setErrorExit; } #Sandbox CRS is up, now simulate an upgrade as below list #1. srvctl upgrade model -firstnode #2. crsctl startupgrade #3. crsctl set crs activeversion (CRSD will terminate) #4. Restart Sandbox CRSD #5. srvctl upgrade model -lastnode $upgradeModelStatus = FAILED; eval { $upgradeModelStatus = $orasrvm->upgrademodel("first"); }; if ($upgradeModelStatus == FAILED) { trace("Dry run...first node upgrade model failed"); goto setErrorExit; } $cmd = "$crsctl startupgrade"; trace ("Invoking \"$cmd\""); @output = system_cmd_capture("$cmd"); $status = shift @output; if ($status == 0) { trace("$cmd.... passed"); } else { goto setErrorExit; } if (((scalar(grep(/1115/, @output)) > 0))) { trace("Upgrade is already completed"); $rc = SUCCESS; goto setErrorExit; } $cmd = "$crsctl set crs activeversion"; trace ("Invoking \"$cmd\""); @output = system_cmd_capture("$cmd"); $status = shift @output; if ($status == 0) { trace("$cmd.... passed"); } else { goto setErrorExit; } if (((scalar(grep(/1115/, @output)) > 0))) { trace("Upgrade is already completed"); $rc = SUCCESS; goto setErrorExit; } if (isOldVersionLT122()) { if (wait_for_sandbox_crsd_exit($crshome) == FAILED) { goto setErrorExit; } if (start_sandbox_crsd($crshome) == FAILED) { goto setErrorExit; } } #print activeversion $cmd = "$crsctl query crs activeversion"; trace ("Invoking \"$cmd\""); @output = system_cmd_capture("$cmd"); $status = shift @output; if ($status == 0) { } else { goto setErrorExit; } $upgradeModelStatus = FAILED; eval { $upgradeModelStatus = $orasrvm->upgrademodel("last"); }; if ($upgradeModelStatus == FAILED) { trace("Dry run...last node upgrade model failed"); goto setErrorExit; } trace("Upgrade dry run completed successfully"); $rc = SUCCESS; setErrorExit: #cleanup $cmd = "$crsctl sandbox clean"; @output = system_cmd_capture("$cmd"); $status = shift @output; trace ("crsctl sandbox clean output = @output"); sb_restore_env_vars(); if ($rc == FAILED) { trace("Upgrade dry run failed"); print_info(694); die(dieformat(362)); } print_info(693); trace("Exiting upgrade dry run"); #cleanup sandbox location my $sbloc = get_crs_sandbox_loc(); trace("Cleaning up sandbox location : $sbloc"); rmtree($sbloc); rmdir($sbloc); if (isOnlyDryRun()) { trace("Exiting upgrade because of checkCrsEntities option"); print_info(695); exit(0); } return SUCCESS; } sub perform_crs_pre_upgrade { my $oldcrshome; my $oldolrlocation; my $crshome = $CFG->ORA_CRS_HOME; my $olrlocation = $CFG->OLR_LOCATION; my $orabase = $CFG->params('ORACLE_BASE'); my $owner = $CFG->params('ORACLE_OWNER'); my $group = $CFG->params('ORA_DBA_GROUP'); my $host = $CFG->HOST; my @old_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')}; my $oldcrsver = join('.',@old_ver); my $oraolr = $CFG->compOLR; my $oraocr = $CFG->compOCR; my $orachm = $CFG->compCHM; my $oraasm = $CFG->compASM; my $old_crs_running; #Find old home $oldcrshome = s_get_olr_file ("crs_home"); if ($oldcrshome eq $crshome) { trace("Upgrade actions are already performed"); return; } if (($CFG->platform_family eq "windows") && ($CFG->isRerun) && (! s_isServiceExists("OracleOHService"))) { # TODO: Add a checkpoint for the ASM rolling migration done by asmca # ohasd was removed in the last run after upgradeASM() or removed manually # during upgradeASM(). # recreate ohasd so that starting old stack would not fail before doing # upgradeASM() again. trace("Re-install ohasd service"); s_register_service('ohasd', $oldcrshome); s_start_service('ohasd', $CFG->params('ORACLE_OWNER'), $oldcrshome); } #do the export of CHA models before the stop of the crs stack if (isFirstNodeToUpgrade()) { my $cc = getClusterClass(); if ($cc eq CLUSTER_CLASS_STANDALONE) { trace("Executing export of CHA user models with orabase - $orabase and crshome $oldcrshome"); # We are going to use old crshome for the binary and new orabase for # storing the exported models if($orachm->exportCHAUserModels($oldcrshome,$orabase, $oldcrsver, $host, $owner, $group) == FAILED) { trace("Failed to export CHA models"); die(dieformat(2605)); } } } #Check if Old Clusterware is running $old_crs_running = check_OldCrsStack(); $oldolrlocation = s_get_olr_file ("olrconfig_loc"); #check validity of olr file before upgrade. Part fix for bug 9466585 if ($oraolr->checkOLR($oldcrshome) != SUCCESS) { die(dieformat(33, $oldcrsver)); } trace("The old crs home = $oldcrshome"); trace("The old OLR location = $oldolrlocation"); if (! $old_crs_running) { trace("Make sure the older stack is completely down"); stopClusterware($oldcrshome, "crs") || die(dieformat(349)); } if (isFirstNodeToUpgrade()) { if (! $old_crs_running) { trace("Old stack is down. Start the stack for rolling or nonrolling migration"); start_OldCrsStack(); $old_crs_running = SUCCESS; } } #do the export of CHA models before the stop of the crs stack if (isFirstNodeToUpgrade()) { my $cc = getClusterClass(); if ($cc eq CLUSTER_CLASS_STANDALONE) { my $ckptName = "ROOTCRS_EXPORTCHA"; if (isCkptSuccess($ckptName)) { trace("CHA models are already exported successfully"); $CFG->wipCkptName("ROOTCRS_STACK"); } else { writeCkpt($ckptName, CKPTSTART); trace("Executing export of CHA user models with orabase - $orabase and crshome $oldcrshome"); # We are going to use old crshome for the binary and new orabase for # storing the exported models if($orachm->exportCHAUserModels($oldcrshome,$orabase, $oldcrsver, $host, $owner, $group) == FAILED) { trace("Failed to export CHA models"); die(dieformat(2605)); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName($ckptName); } } } if (isFirstNodeToUpgrade()) { #Exporting RHP repository trace("Calling SRVM upgrade for first node"); ($CFG->compSRVM)->upgradeCurrentNode() || die(dieformat(645)); } # Stop the mgmtdb after exporting the RHP data. if (! $CFG->SIHA) { # the GIMR is upgraded from 12201 onwards, whereas it is recreated for # upgrades from other releases. Hence disable GIMR clients for upgrades # from 12201 onwards. if (! isOldVersionLT122) { disableGIMRClients() || die(dieformat(678));; } # the GIMR is either recreated or upgraded based on the old version. # It needs to be disabled either way. disableGimrDb(); } trace("Upgrade ASM in a cluster"); my $ckptName = "ROOTCRS_UPGRADEASM"; my $ckptStatus = CKPTSTART; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); } trace("'$ckptName' state is $ckptStatus"); if ($ckptStatus eq CKPTSUC) { trace("ASM has already been upgraded"); $CFG->wipCkptName("ROOTCRS_STACK"); } else { writeCkpt($ckptName, CKPTSTART); $CFG->wipCkptName($ckptName); if (isFirstNodeToUpgrade()) { set_flag_for_ocr_changed(); # Set clusterware to rolling migration mode for Far ASM. For other ASM # modes, it is set by ASM module. if ( (isFarASM()) && (isRolling()) && (! isOldVersionLT121()) ) { print_info(524); $oraasm->startRollingUpgrade() || die(dieformat(511)); print_info(525); } } $oraasm->upgradeCurrentNode($oraasm->UPGRADE_ASM); writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName("ROOTCRS_STACK"); } if ((isFirstNodeToUpgrade()) && (isLegacyASM())) { trace("Disable all ACFS & volume resources before stopping " . "the older stack"); if ($CFG->ACFS_ADVM_RESOURCES_LIST) { my $acfs_res_list = $CFG->ACFS_ADVM_RESOURCES_LIST; trace("ACFS & ADVM resources to be disabled: [$acfs_res_list]"); my @acfsres = split(/,/, $acfs_res_list); set_flag_for_ocr_changed(); foreach my $acfs (@acfsres) { disableRes($acfs, $CFG->HOST, (isOldVersionLT12102() ? FALSE : TRUE), $CFG->OLD_CRS_HOME) || die(dieformat(679)); } } } # Stop and disable OC4J on first node if (isFirstNodeToUpgrade()) { trace("Stopping and disabling OC4J resource from old CRS home"); set_flag_for_ocr_changed(); stop_and_disable_J2EEContainer($CFG->OLD_CRS_HOME); } print_info(466); # As a workaround for bug#26979409, which has been fixed in 18.1 but not # been backported to 12.2.0.1 yet, here the rhpserver is explicitly stopped # before the older stack gets shut down. if (isFirstNodeToUpgrade() && (! isOldVersionLT122())) { trace("Attempt to stop the rhpserver before stopping the older stack"); my $cmd = "stop rhpserver"; my @output; my $rc = srvctl_capture(FALSE, \@output, $cmd, $oldcrshome); trace("rc=$rc"); trace(@output); } # stop the stack if it is already running. if (! stopClusterware($oldcrshome, "crs")) { die(dieformat(349)); } print_info(467); if ($CFG->platform_family eq "windows") { s_unregister_service("ohasd"); s_upgrade_services(); if ((! is_dev_env()) && (isRolling())) { if (! upgrade_asm_service()) { die(dieformat(307)); } } } $oraocr->s_backupocrloc(); if (isFirstNodeToUpgrade ()) { #copy necessary files to new home copyfiles($oldcrshome, $crshome); #copy olr file to new home and set permissions copy_file($oldolrlocation, $olrlocation); } elsif (!isHomeShared()) { #copy gpnp global files to new home copyfiles($oldcrshome, $crshome); #copy olr file to new home and set permissions copy_file($oldolrlocation, $olrlocation); } s_set_ownergroup ($CFG->SUPERUSER, $CFG->params('ORA_DBA_GROUP'), $olrlocation); s_set_perms ("0600", $olrlocation); # For handling CHM Upgrade. Bug 11852891. if (!$orachm->perform_CHM_upgrade()) { trace("Failed to perform Cluster Health Monitor upgrade"); die(dieformat(404)); } # copy/generate cluster root wallet in new installation if ($CFG->isFirstNodeToUpgrade) { trace("Placing cluster root wallet in new Oracle Base location..."); my $root_wallet = catfile('security', 'rootwallet', 'cwallet.sso'); my $old_rootwallet_path = catfile(get_configured_orabase($oldcrshome), 'crsdata', $host, $root_wallet); my $new_rootwallet_path = catfile(get_configured_orabase($crshome), 'crsdata', $host, $root_wallet); # cluster root certificate could not be retrieved from old installation; # generate brand new cluster root certificate. if (!-e $old_rootwallet_path) { generate_root_wallet($crshome); } # old cluster root certificate exists, but ORACLE_BASE location has # changed; copy the existing root certificate to the new location elsif ($old_rootwallet_path ne $new_rootwallet_path) { copy_file($old_rootwallet_path, $new_rootwallet_path); } trace("Cluster root certificate has been placed in $new_rootwallet_path"); # push to rest of nodes; it will die if any copy fails push_crsdata_file_clusterwide($root_wallet, $crshome); } } sub perform_siha_pre_upgrade { my $oldcrshome; my $oldolrlocation; my $crshome = $CFG->ORA_CRS_HOME; my $olrlocation = $CFG->OLR_LOCATION; my $host = $CFG->HOST; my $success; my @old_ver = @{$CFG->oldconfig('ORA_CRS_VERSION')}; my $oldcrsver = join('.',@old_ver); my $oraolr = $CFG->compOLR; my $oraocr = $CFG->compOCR; my $orachm = $CFG->compCHM; #Find old home $oldcrshome = s_get_olr_file ("crs_home"); if ($oldcrshome eq $crshome) { trace("Upgrade actions are already performed"); return; } if (($CFG->platform_family eq "windows") && ($CFG->isRerun) && (! s_isServiceExists("OracleOHService"))) { # TODO: Add a checkpoint for the ASM rolling migration done by asmca # # ohasd was removed in the last run after upgradeASM() or removed manually # during upgradeASM(). # recreate ohasd so that starting old stack would not fail before doing # upgradeASM() again. trace("Re-install ohasd service"); s_register_service('ohasd', $oldcrshome); s_start_service('ohasd', $CFG->params('ORACLE_OWNER'), $oldcrshome); } $oldolrlocation = s_get_olr_file ("olrconfig_loc"); #check validity of olr file before upgrade. Part fix for bug 9466585 if ($oraolr->checkOLR($oldcrshome) != SUCCESS) { die(dieformat(33, $oldcrsver)); } trace("The old crs home = $oldcrshome"); trace("The old OLR location = $oldolrlocation"); if (isASMConfigured($oldcrshome)) { if (!perform_sihaASM_upg($oldcrshome, $oldcrsver)) { die(dieformat(304)); } } # CSSD and DISKMON should not be disabled for upgrades # from 12.1.0.1 and later. if (isOldVersionLT121()) { trace("Disabling ora.cssd and ora.diskmon"); # Disable ora.cssd my $crsctl = catfile($CFG->ORA_CRS_HOME, 'bin', 'crsctl'); my @cmd = ($crsctl, 'modify', 'res', 'ora.cssd', '-attr', "\"ENABLED=0\"", '-init'); my $status = system_cmd(@cmd); if (0 != $status) { die("Failed to disable CSSD startup"); } if ($CFG->platform_family ne "windows") { # Disable ora.diskmon @cmd = ($crsctl, 'modify', 'res', 'ora.diskmon', '-attr', "\"ENABLED=0\"", '-init'); $status = system_cmd(@cmd); if (0 != $status) { die("Failed to disable DISKMON startup"); } } } if (! stopOracleRestart()) { die(dieformat(348)); } if ($CFG->platform_family eq "windows") { s_unregister_service("ohasd"); if ((! is_dev_env()) && (isRolling())) { if (! upgrade_asm_service()) { die(dieformat(307)); } } } $oraocr->s_backupocrloc(); #copy necessary files to new home copyfiles($oldcrshome, $crshome); copy_gpnpnodefiles($oldcrshome, $crshome); #copy olr file to new home and set permissions copy_file($oldolrlocation, $olrlocation); s_set_ownergroup ($CFG->params('ORACLE_OWNER'), $CFG->params('ORA_DBA_GROUP'), $olrlocation); s_set_perms ("0600", $olrlocation); } #------------------------------------------------------------------------------- # Function: checks if the user running the upgrade matches with the owner # of the old crs home. # Args : none # Returns : TRUE or FALSE #------------------------------------------------------------------------------- sub checkOldCrsOwner { my $ch = $CFG->OLD_CRS_HOME; my $cssd_bin = "$ch/bin/ocssd.bin"; my $old_ch_owner = getBinaryOwner($cssd_bin); my $ch_owner = $CFG->params('ORACLE_OWNER'); trace("new CH owner = $ch_owner"); trace("old CH owner = $old_ch_owner"); if ($old_ch_owner != $ch_owner) { print_error(360, $old_ch_owner, $ch_owner); return FALSE; } return TRUE; } #------------------------------------------------------------------------------ # Function: Gets the OS owner of the given binary. # Args : the name of the binary # Returns : owner name. #------------------------------------------------------------------------------- sub getBinaryOwner { my $binary = $_[0]; my $binary_o; if ( -f $binary ) { my ($dev, $ino, $mode, $nlink, $uid_stat, $gid, $rdev, $size, $atime, $mtime, $ctime, $blksize, $blocks) = stat( $binary ); my ($name, $passwd, $uid, $ugid, $quota, $comment, $gcos, $dir, $shell) = getpwuid( $uid_stat ); if (not defined $name) { print_error(359,$binary); } else { $binary_o = $name; trace( "Owner of executable $binary:: $binary_o" ); } } else { print_error(13, $binary); } return $binary_o; } sub perform_sihaASM_upg #------------------------------------------------------------------------------- # Function: upgrade siha asm # Args : [0] - old crs home # [1] - old crs version # Returns : TRUE if success # FALSE if failed #------------------------------------------------------------------------------- { my $oldhome = $_[0]; my $oldver = $_[1]; my $success = TRUE; my $status; my @runasmca; my $asmcaArgs = "-silent -upgradeOracleRestartASM -oldCRSHome $oldhome " ."-oldCRSVersion $oldver"; my $ckptName = "ROOTHAS_UPGRADEASM"; if (isCkptSuccess($ckptName)) { trace("SIHA ASM has already been upgraded"); $CFG->wipCkptName("ROOTHAS_STACK"); return TRUE; } else { writeCkpt($ckptName, CKPTSTART); trace("Start ASM upgrade for Oracle Restart"); @runasmca = (catfile ($CFG->ORA_CRS_HOME, "bin", "asmca"), $asmcaArgs); trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runasmca"); $status = run_as_user($CFG->params('ORACLE_OWNER'), @runasmca); if ($status != 0) { trace("upgrade asm for Oracle Restart failed..."); writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTHAS_STACK"); $success = FALSE; print_error(164); } else { trace("upgrade asm for Oracle Restart suceeded."); writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName("ROOTHAS_STACK"); } return $success; } } sub isFirstNodeToUpgrade { my $ret; trace ("isFirstNodeToUpgrade..."); if ($CFG->JOIN) { trace("Join an upgraded cluster"); return FALSE; } # This function is frequently invoked on each node during upgrade, # we cache the value of $firstnode_to_upgrade the first time this # function is executed, so that we don't have to go through the # whole function each time. if (defined $CFG->isFirstNodeToUpgrade) { my $cachedValue = $CFG->isFirstNodeToUpgrade; trace ("isFirstNodeToUpgrade: $cachedValue"); return $CFG->isFirstNodeToUpgrade; } $ret = isForcedFirstNode(); trace("isFirstNodeToUpgrade: $ret"); $CFG->isFirstNodeToUpgrade($ret); return $ret; } sub upgrade_asm_service { my $success = TRUE; # Do not change the order of these parameters as asmca requires the # parameters to be in a specific order or it will fail my @asmca = (catfile ($CFG->ORA_CRS_HOME, "bin", "asmca"), '-silent', '-upgradeLocalASM', '-winSvc', $CFG->params('ASMCA_ARGS')); trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @asmca"); my $status = run_as_user($CFG->params('ORACLE_OWNER'), @asmca); if ($status != 0) { $success = FALSE; print_error(162); } return $success; } sub upgrade_config { my $crsctlbin = crs_exec_path('crsctl'); my $success = FAILED; my $status; my $cmd; my @output; # Tell crs subsystem to copy the old resource profiles to new # engine. Which does not copy the nodeapps. $cmd = "$crsctlbin startupgrade"; trace ("Invoking \"$cmd\""); @output = system_cmd_capture("$cmd"); $status = shift @output; # Do not print the output of 'crsctl startupgrade' to the console if # it fails due to CRS stack not yet coming up in a rerun case if (0 == $status) { trace("Successfully completed 'crsctl startupgrade'"); $success = SUCCESS; print_lines(@output); } else { if ($CFG->isRerun) { trace("Sleep unitl CRSD comes up"); for (my $i = 0; $i < 36; $i++) { sleep(10); if (GI_STACK_UP == checkGIStack()) { $status = system_cmd("$cmd"); if (0 == $status) { trace("Succeeded in executing 'crsctl startupgrade' during rerun"); $success = SUCCESS; last; } } } if (GI_STACK_UP != checkGIStack()) { die(dieformat(251)); } } else { trace("Failed to execute 'crsctl startupgrade'"); $success = FAILED; print_lines(@output); } } return $success; } # Get CRS active version major number (i.e. 10, 11, etc.) Stack must be up. sub getCRSMajorVersion { my $crsctlbin = crs_exec_path('crsctl'); my $ver = 0; my @cmd = ($crsctlbin, 'query', 'crs', 'activeversion'); my @out = system_cmd_capture(@cmd); my $rc = shift @out; if ($rc == 0) { my $verinfo = getVerInfo($out[0]); my @versionarr = split(/\./, $verinfo); $ver = $versionarr[0]; trace("crs major version=$ver"); } else { my $mycmd = join(' ', @cmd); print_error(180, $mycmd); trace ("@cmd ... failed rc=$rc with message:\n @out \n"); } return $ver; } sub upgrade_node { # start checkpoint my $ckptStatus; my $ckptName = "ROOTCRS_NODECONFIG"; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); trace("'$ckptName' state is $ckptStatus"); if (($ckptStatus eq CKPTSUC) && (! $CFG->FORCE)) { trace("CRS node configuration resources are already configured"); $CFG->wipCkptName("ROOTCRS_STACK"); return SUCCESS; } } writeCkpt($ckptName, CKPTSTART); $CFG->wipCkptName($ckptName); $CFG->compACFS(oraClusterwareComp::oraacfs->new("ACFS")); # upgrade node my $success = TRUE; my $orasrvm = $CFG->compSRVM; trace("Performing upgraded node configurations"); if (isFirstNodeToUpgrade()) { trace("Performing first node operations"); if (FAILED == $orasrvm->postUpgradeFirstNode()) { writeCkpt($ckptName, CKPTFAIL); exit(-1); } # Get ACFS/ADVM resources re-enabled and online since all dependencies # have been set up and ready if ((isLegacyASM()) && ($CFG->ACFS_ADVM_RESOURCES_LIST)) { trace("Re-enable and start ACFS/ADVM resources on the first node"); ($CFG->compACFS)->enableAndStartADVMACFSonNode(); } } if (($success == TRUE) && isLastNodeToUpgrade()) { trace("Performing last node operations"); my $acfsreslist = ''; $acfsreslist = ($CFG->compACFS)->getAllACFSandADVMResInCluster($CFG->ORA_CRS_HOME); if ($acfsreslist ne ''){ # CRS does not allow resource operations between 'crsctl startupgrade' # and 'crsctl set activeversion', so start the proxy before that. trace("Enable and start proxy for Acfs"); srvctl(TRUE, "enable asm -proxy", $CFG->ORA_CRS_HOME) || die(dieformat(372)); srvctl(TRUE, "start asm -proxy", $CFG->ORA_CRS_HOME) || die(dieformat(373)); } $success = upgrade_config(); if (!$success) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(454)); } # Setting active version can handle ASM stop rolling migration $success = setActiveversion(); if (!$success) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(281)); } if ((! $orasrvm->postUpgradeLastNode()) || (! modify102Resources())) { writeCkpt($ckptName, CKPTFAIL); exit(-1); } # Bug 17778985. Reset cardinality of Flex ASM on last node # when upgrading from 12.1 if ( (isRolling()) && isNearASM() && isOldVersion121() ) { $success = resetASM($CFG->ORA_CRS_HOME, $ckptName); if (!$success) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(514)); } } if (($CFG->params('MGMT_DB') =~ m/true/i) && (! isRemoteGIMR()) && (lc(getClusterClass()) ne lc(CLUSTER_CLASS_MEMBER)) && (!add_mgmt_db_listener())) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(437)); } # Configure CHA only where CHA is supported if (isCHASupported()) { # If old version is earlier than 12.1, CHA can be forcibly # configured and need to ignore start errors as MGMTDB has # not been configured yet # If old version is 12.1, MGMTDB resource gets deleted on # the first node # Pass '-force' for all upgrade scenarios in 12.1.0.2.0 if (!add_cha('-force')) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(2602)); } # Set the resource target to ONLINE state. if (!start_cha(TRUE, "")) { writeCkpt($ckptName, CKPTFAIL); die(dieformat(2604)); } } # Upgrade ADVM volumes and ACFS registry ($CFG->compACFS)->upgradeCurrentNode(($CFG->compACFS)->UPGRADE_ADVM_REGISTRY); # Add RHPServer for Standalone Cluster my $cluster_class = lc(trim($CFG->params('CLUSTER_CLASS'))); my $isStandaloneCluster = ($cluster_class eq lc(CLUSTER_CLASS_STANDALONE)); my $ora_add_rhps = $ENV{'_ORA_ADD_RHPS'}; trace("Env var '_ORA_ADD_RHPS': $ora_add_rhps"); if (! (is_dev_env() && (lc($ora_add_rhps) eq "false")) && ($CFG->platform_family eq "unix") && (! isRHPProvisioned()) && $isStandaloneCluster && $CFG->compACFS->isACFSSupported() && (isOldVersionLT122() || isMgmtdbConfigured()) && ! isRHPSConfigured() && ! isRHPCConfigured()) { my $globalCkpt = 'ROOTCRS_ADDRHPS'; # this ckpt is used for a following downgrade instead of for upgrade rerun. my $wipCkpt = $CFG->wipCkptName; $CFG->wipCkptName($globalCkpt); writeGlobalCkpt($globalCkpt, CKPTSTART, "-transferfile"); if (($orasrvm->add_rhpserver("-local")) && (! isCkptSuccess($globalCkpt, "-global"))) { writeGlobalCkpt($globalCkpt, CKPTSUC, "-transferfile"); } $CFG->wipCkptName($wipCkpt); } if (isOldVersionLT122 ()) { my $cvuCheckInterval = getCkptPropertyValue("ROOTCRS_OLDHOMEINFO", "CVU_CHECK_INTERVAL", "-global"); if (!isCkptPropertyExists ($ckptName, "IS_NEW_CVU_ADDED")) { if (add_CVU($cvuCheckInterval)) { trace ("CVU resource successfully added"); writeCkptProperty ($ckptName, "IS_NEW_CVU_ADDED", TRUE); writeCkptProperty("ROOTCRS_OLDHOMEINFO", "IS_CVU_CONFIGURED", 1, "-global", "-transferfile"); } else { trace("Failed to add CVU resource"); $success = FALSE; } } if (!isCkptPropertyExists ($ckptName, "IS_CVU_RUNNING")) { if (start_CVU()) { trace ("CVU resource successfully upgraded"); writeCkptProperty ($ckptName, "IS_CVU_RUNNING", TRUE); } else { trace("Failed to upgrade CVU resource"); $success = FALSE; } } } else { $success = enable_CVU(); if ($success) { $success = start_CVU(); } } if ( $success && ($CFG->params('ORACLE_OWNER') ne getLsnrUsername()) ) { upgradeNodeListener(); } } # end isLastNodeToUpgrade() if (($success == TRUE) && (! isFirstNodeToUpgrade()) && (! isLastNodeToUpgrade())) { $success = ($CFG->compSRVM)->postUpgradeMiddleNode(); } upgradeMgmtdb() if ($success && isMgmtdbConfigured()); if ($success) { writeCkpt($ckptName, CKPTSUC); } else { writeCkpt($ckptName, CKPTFAIL); (isOldVersionLT121 ()) ? die(dieformat(565)) : die(dieformat(566)); } return $success; } sub upgradeNodeListener { if ($CFG->platform_family eq "windows") { trace("No role separation for listeners on Windows"); return; } print_info(480); upgradeRemovedListener(); my $ckptName = "ROOTCRS_NODECONFIG"; my %listeners = getCurrentNetLsnrs(isOldVersionLT121() ? ($CFG->OLD_CRS_HOME) : ($CFG->ORA_CRS_HOME)); foreach my $lsnr (keys %listeners) { my $lsnr_home = $listeners{$lsnr}->{'lsnr_home'}; my $lsnr_port = $listeners{$lsnr}->{'port'}; trace("Trying to upgrade listener:"); if (isCkptPropertyExists($ckptName, $lsnr)) { trace("Listener '$lsnr' has been upgraded, hence bypassing ..."); next; } my $srvctl_home; if (isOldVersionLT121()) { $srvctl_home = $lsnr_home; } else { $srvctl_home = $CFG->ORA_CRS_HOME; } stopNetLsnr($srvctl_home, $lsnr); my $lsnrProperty = $lsnr.":".$lsnr_home.":".$lsnr_port; removeNetLsnr(isOldVersionLT121() ? ($CFG->OLD_CRS_HOME) : ($CFG->ORA_CRS_HOME), $lsnr); writeCkptProperty($ckptName, "LISTENER_REMOVED_NOT_UPGRADED", $lsnrProperty); upgradeNetLsnrWithUsername(getLsnrUsername(),$lsnr_home,$lsnr,$lsnr_port); startNetLsnr(FALSE, getLsnrUsername(), $lsnr); writeCkptProperty($ckptName, "LISTENER_UPGRADE_COMPLETED", $lsnr); writeCkptProperty($ckptName, $lsnr, "true"); } print_info(481); } sub upgradeRemovedListener { my $ckptName = "ROOTCRS_NODECONFIG"; if (isCkptPropertyExists($ckptName, "LISTENER_REMOVED_NOT_UPGRADED")) { my $lsnrProperty = trim(getCkptPropertyValue($ckptName, "LISTENER_REMOVED_NOT_UPGRADED")); my $lsnr = (split(/:/, $lsnrProperty))[0]; my $lsnrHome = (split(/:/, $lsnrProperty))[1]; my $lsnrPort = (split(/:/, $lsnrProperty))[2]; if ((! isCkptPropertyExists($ckptName, "LISTENER_UPGRADE_COMPLETED")) || ($lsnr ne trim(getCkptPropertyValue($ckptName, "LISTENER_UPGRADE_COMPLETED")))) { trace("Proceed with upgrading the node listener '$lsnr'. Listener home: $lsnrHome"); upgradeNetLsnrWithUsername(getLsnrUsername(), $lsnrHome, $lsnr, $lsnrPort); startNetLsnr(FALSE, getLsnrUsername(), $lsnr); writeCkptProperty($ckptName, "LISTENER_UPGRADE_COMPLETED", $lsnr); writeCkptProperty($ckptName, $lsnr, "true"); } } } sub setActiveversion { print_info(478); my $crsctl = crs_exec_path('crsctl'); my @cmd; if ($CFG->FORCE) { @cmd = ($crsctl, 'set', 'crs', 'activeversion', '-force'); } else { @cmd = ($crsctl, 'set', 'crs', 'activeversion'); } my $cmdStr = join(' ', @cmd); print_info(482, $cmdStr); my $status = system (@cmd); if (0 == $status) { trace ("@cmd ... passed"); sleep(60); # Wait until CRS changes to new engine trace("Check if the stack restarted after the AV change"); if (!check_service("crs", 120)) { trace("The stack did not come up after the AV change"); die(dieformat(251)); } } else { my $mycmd = join(' ', @cmd); trace("$mycmd failed with status $status"); print_error(180, $mycmd); return FAILED; } print_info(479); return SUCCESS; } sub isLastNodeToUpgrade { trace ("isLastNodeToUpgrade..."); if (($CFG->FORCE) && (CKPTSUC eq getCkptStatus("ROOTCRS_FIRSTNODE", "-global"))) { trace("Force upgrade invoked"); $CFG->isLastNodeToUpgrade(TRUE); writeGlobalCkpt("ROOTCRS_LASTNODE", CKPTSTART, "-transferfile") if (CKPTSUC ne getCkptStatus("ROOTCRS_LASTNODE", "-global")); return TRUE; } if ($CFG->JOIN) { trace("Join an upgraded cluster"); $CFG->isLastNodeToUpgrade(FALSE); return FALSE; } if (defined $CFG->isLastNodeToUpgrade) { my $cachedValue = $CFG->isLastNodeToUpgrade; trace("isLastNodeToUpgrade: $cachedValue"); if ($CFG->isLastNodeToUpgrade == TRUE) { writeGlobalCkpt("ROOTCRS_LASTNODE", CKPTSTART, "-transferfile"); } return $CFG->isLastNodeToUpgrade; } my $lastnode_to_upgrade = TRUE; my $crsctl = crs_exec_path("crsctl"); # get current releaseversion open (QUERYCRS, "$crsctl query crs releaseversion |"); my $output = ; close (QUERYCRS); my $release_version = getVerInfo($output); trace ("release_version=$release_version"); # get current softwareversion my $nodelist = $CFG->oldconfig('NODENAME_LIST'); my @nodes = split (/,/, $nodelist); my $software_version; my $host = tolower_host(); foreach my $nodename (@nodes) { if (lc($nodename) =~ /\b$host\b/i) { next; } $software_version = getcursoftversion($nodename); trace ("software_version on $nodename=$software_version"); # compare version if ($software_version ne $release_version) { $lastnode_to_upgrade = FALSE; last; } } trace("last node to upgrade is $lastnode_to_upgrade"); # When upgrade runs in parallel on all non-first nodes, there could be one # or more nodes where upgrade gets rejected as CRSD is upgrading resources # on the last node. Later on users re-run upgrade on the failed nodes, # those nodes should not be mistakenly considered as the last node. my $lastnodeState = trim(getCkptStatus("ROOTCRS_LASTNODE", "-global")); my $locCkptStatus = trim(getCkptStatus("ROOTCRS_STACK")); if (($lastnodeState eq CKPTSUC) && ($locCkptStatus ne CKPTSUC)) { trace("Last node actions have been done"); $lastnode_to_upgrade = FALSE; } if (TRUE == $lastnode_to_upgrade) { $lastnode_to_upgrade = isLastNodeinOCR(); } $CFG->isLastNodeToUpgrade($lastnode_to_upgrade); if ($CFG->isLastNodeToUpgrade == TRUE) { writeGlobalCkpt("ROOTCRS_LASTNODE", CKPTSTART, "-transferfile"); } return $lastnode_to_upgrade; } sub isLastNodeinOCR { my $ocrkey = 'LASTNODE'; my $ocrval = $CFG->HOST; my $oraocr = $CFG->compOCR; my $succ = $oraocr->createAndSetOcrKeyPair($ocrkey, $ocrval, $CFG->ORA_CRS_HOME); if (! $succ) { die(dieformat(608, "SYSTEM.$ocrkey")); } $ocrval = ""; ($succ, $ocrval) = $oraocr->getOcrKeyValue($ocrkey, $CFG->ORA_CRS_HOME); if (! $succ) { die(dieformat(662, "SYSTEM.$ocrkey")); } if ($ocrval eq $CFG->HOST) { trace("The OCR key [SYSTEM.$ocrkey] points to the current node"); return TRUE; } return FALSE; } sub vipInfo { my @crs_nodevip_list_old; my @crs_version = @{$CFG->oldconfig('ORA_CRS_VERSION')}; if (($crs_version[0] eq '10' && $crs_version[1] eq '1') && (! isNodeappsExists())) { my $crs_nodevips = $CFG->params('CRS_NODEVIPS'); if ($crs_nodevips eq "") { print_error(397, $CFG->ORA_CRS_HOME); exit 1; } else { $crs_nodevips =~ s/'//g; # ' in comment to avoid confusion of editors. $crs_nodevips =~ s/"//g; # remove " on Windows $crs_nodevips =~ s/\/\*/\//g; # handle different interface case @crs_nodevip_list_old = split (/\s*,\s*/, $crs_nodevips); } } else { if ($CFG->platform_family eq "windows") { @crs_nodevip_list_old = get_OldVipInfoFromOCRDump(); } else { # Retrieve VIP config info from new stack, # though we still call binaries from old stack my $env = $ENV{'SRVM_TRACE'}; undef $SRVM_TRACE; @crs_nodevip_list_old = get_OldVipInfo(); $ENV{'SRVM_TRACE'} = $env; } } trace("crs_nodevip_list_old = [@crs_nodevip_list_old]"); return @crs_nodevip_list_old; } sub is112ASMExists #------------------------------------------------------------------------------- # Function: Check if 11.2 ASM exists # Args : none # Returns : TRUE if exists # FALSE if not exists #------------------------------------------------------------------------------- { if (check_service("ora.asm", 1)) { return TRUE; } else { return FALSE; } } sub update_ASMowner #------------------------------------------------------------------------------- # Function: update ASM owner to grid owner # Args : none # Returns : none #------------------------------------------------------------------------------- { trace("Updating ASM owner if required"); my $CRSCTL = catfile ($CFG->params('ORACLE_HOME'), 'bin', 'crsctl'); my $rc; my @out; my $ohown = $CFG->params('ORACLE_OWNER'); my $cmd; $cmd = "$CRSCTL stat resource -w 'NAME en .asm'"; @out = system_cmd_capture("$cmd"); $rc = shift @out; my @cmdout = grep(/NAME/, @out); trace("ASM resource list is @cmdout"); foreach my $str (@cmdout) { my ($tag, $resname) = split(/=/, $str); chomp ($resname); if ($resname =~ "ora.asm") { trace("skip update owner for ora.asm resource"); next; } $cmd = "$CRSCTL getperm resource $resname"; @out = system_cmd_capture("$cmd"); $rc = shift @out; my @cmdout = grep(/^owner:/, @out); trace("$resname ACL is @cmdout"); my @val = split(/:/, $cmdout[0]); my $asmown = $val[1]; chomp ($asmown); trace("asm owner is $asmown"); if ($asmown eq $ohown ) { trace("ASM resource owner is the same as grid owner.Nothing to be done"); next; } $cmd = "$CRSCTL setperm resource $resname -o $ohown -unsupported"; @out = system_cmd_capture("$cmd"); $rc = shift @out; if ($rc == 0) { trace ("Successfully updated owner for resource $resname"); } else { error ("Failed to update owner for resource $resname"); } } } sub setASMNetworks { my $rc; my @getif_out; my @subnets; trace("Set ASM networks for upgrading from legacy ASM"); ($rc, @getif_out) = get_oifcfg_info(($CFG->ORA_CRS_HOME, 'getif')); die(dieformat(309, join(' ', @getif_out))) if ($rc != 0); # The modify_networks() finds all the private networks from an array # of oifcfg-style output, and sets them with the network type # "cluster_interconnect,asm" @subnets = modify_networks(@getif_out); # Create a space-separated list that is passed to oifcfg if (scalar(@subnets) > 0) { my $asmNetworks = join(" ", sort(@subnets)); setNetworkInterface($asmNetworks) || die(dieformat(654)); } } sub upgrade_clscfg { my $ckptName; my $ckptStatus; $ckptName = ($CFG->FORCE) ? "ROOTCRS_CLSCFGUPG_FORCE" : "ROOTCRS_CLSCFGUPG"; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); trace("'$ckptName' state is $ckptStatus"); if (CKPTSUC eq $ckptStatus) { trace("Already updated node specific version and patch level keys"); $CFG->wipCkptName("ROOTCRS_STACK"); if (isLastNodeToUpgrade()) { run_crs_cmd('oifcfg', 'setif', '-global'); if (isLegacyASM()) { setASMNetworks(); } } return; } } writeCkpt($ckptName, CKPTSTART); $CFG->wipCkptName($ckptName); # execute clscfg -upgrade my $status; my $asmgrp = $CFG->params('ORA_ASM_GROUP'); my $clscfg = catfile($CFG->ORA_CRS_HOME, 'bin', 'clscfg'); my $cmd = "$clscfg -upgrade -g " . $asmgrp; # Add site name to site GUID configuration on the first node during upgrade if ((isOldVersionLT122()) && (defined($CFG->clscfg_site2guid_arg_y))) { $cmd = $cmd . " " . $CFG->clscfg_site2guid_arg_y; } if (defined($CFG->clscfg_clsprop_ocr_arg_p)) { $cmd = $cmd . " " . $CFG->clscfg_clsprop_ocr_arg_p; } if ($CFG->FORCE) { $cmd = $cmd . " -lastnode"; } $status = system_cmd($cmd); if ($status == 0) { writeCkpt($ckptName, CKPTSUC); trace ("clscfg -upgrade completed successfully"); } else { writeCkpt($ckptName, CKPTFAIL); trace("clscfg -upgrade failed with status $status"); print_error(180, "clscfg -upgrade"); exit 1; } # execute 'oifcfg setif -global' if (isLastNodeToUpgrade()) { $status = run_crs_cmd('oifcfg', 'setif', '-global'); if (isLegacyASM()) { setASMNetworks(); } } } sub ModActionScript #------------------------------------------------------------------------------- # Function: Modify all db resources ACTION_SCRIPT and set it to %crs_home% # Args : none #------------------------------------------------------------------------------- { trace("Modify ACTION_SCRIPT"); # get db resources my $crsctl = crs_exec_path('crsctl'); my @cmd = ($crsctl, 'stat', 'res', '-w', '"((TYPE = application) AND (NAME en .db) AND (NAME st ora.))"'); my @output = system_cmd_capture(@cmd); my $rc = shift @output; my @resname = grep(/NAME=/, @output); trace("output=@output"); trace("resname=@resname"); # modify ACTION_SCRIPT my ($racgwrap, $dummy, $res); if ($CFG->platform_family eq "windows") { $racgwrap = catfile('%CRS_HOME%', 'bin', 'racgwrap.bat'); } else { $racgwrap = catfile('%CRS_HOME%', 'bin', 'racgwrap'); } foreach (@resname) { ($dummy, $res) = split ('NAME='); @cmd = ($crsctl, 'modify', 'res', $res, '-attr', "\"ACTION_SCRIPT=$racgwrap\"", '-unsupported'); $rc = system_cmd(@cmd); if ($rc == 0) { trace("@cmd ... success"); } else { trace("@cmd ... failed"); } } } sub modifyClusterName { trace("modify CLUSTER_NAME in crsconfig_params"); my $params_file = $CFG->paramfile; my @params_table = read_file ($params_file); my $updateParamsFile = FALSE; my $ix = 0; foreach my $rec (@params_table) { chomp($rec); if ($rec =~ m/^CLUSTER_NAME=/) { my ($key, $value) = split (/=/, $rec); if (! $value) { my $cluster_name = getClusterName(); if ($cluster_name) { $params_table[$ix] = 'CLUSTER_NAME=' . "$cluster_name"; $updateParamsFile = TRUE; last; } } } $ix++; } if ($updateParamsFile) { # save original params file my $save_file = catfile (dirname($params_file), 'crsconfig_params.saved'); copy_file ($params_file, $save_file); # delete old file and create new file if (s_remove_file($params_file)) { open (SFILE, ">$params_file") || die(dieformat(207, $params_file, $!)); foreach my $rec (@params_table) { chomp($rec); print SFILE "$rec\n"; } close SFILE; } } } sub getNotupgradedNodes { trace("Find out which nodes didn't get upgraded"); my @notupgradedNodes; my $softver; my $relver = getcrsrelver($CFG->ORA_CRS_HOME); my $nodelist = $CFG->oldconfig('NODENAME_LIST'); my @clunodes = split(',', $nodelist); foreach my $clunode (@clunodes) { if ($CFG->HOST =~ /^$clunode$/i) { next; } $softver = getcursoftversion($clunode); if (!isVersionMatch($relver, $softver)) { push(@notupgradedNodes, $clunode); } } trace("Not upgraded nodes: @notupgradedNodes"); return @notupgradedNodes; } # For a force upgrade case, remove the ASM node resources of un-upgraded nodes # so that ASMCA can succeed after rootupgrade.sh sub removeASMNodeRes { my $ckptStatus; my $ckptName = "ROOTCRS_FORCEUPG_RMASMRES"; if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); trace("'$ckptName' state is $ckptStatus"); if (($ckptStatus eq CKPTSUC) && ($CFG->FORCE)) { trace("Removal of ASM node resources of un-upgraded nodes is already done"); $CFG->wipCkptName("ROOTCRS_STACK"); return SUCCESS; } } writeCkpt($ckptName, CKPTSTART); $CFG->wipCkptName($ckptName); my %nodeNumHash; my @nodeNumArray = split(/,/, getNodeNumList($CFG->ORA_CRS_HOME)); foreach my $elem (@nodeNumArray) { my @val = split(/:/, $elem); $nodeNumHash{${val[0]}} = $val[1]; } my $crsctl = crs_exec_path('crsctl'); my @notupgradedNodes = getNotupgradedNodes(); foreach my $node (@notupgradedNodes) { my $nodeNum = $nodeNumHash{$node}; my $resName = "ora\.".$node."\.ASM$nodeNum"."\.asm"; my @out = system_cmd_capture($crsctl, "stat", "resource", $resName); my $asmhome = crsctlRemoveASM($resName, $node); srvctlRemoveASM($resName, $node, $asmhome); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName("ROOTCRS_STACK"); return SUCCESS; } sub crsctlRemoveASM { my $resName = $_[0]; my $node_name = $_[1]; my $ckptName = "ROOTCRS_FORCEUPG_RMASMRES"; my $asmhome; trace("Romve ASM resource <$resName> on node <$node_name>"); my $crsctl = crs_exec_path('crsctl'); my @output = system_cmd_capture($crsctl, "stat", "resource", $resName); my $rc = shift(@output); if ((0 == $rc) && (scalar(grep(/CRS-2613/, @output)) > 0)) { trace("ASM resource <$resName> does not exist"); } else { trace("Trying to retrieve ASM home for <$resName>"); @output = system_cmd_capture($crsctl, "stat", "res", $resName, "-p"); $rc = shift(@output); if (0 != $rc) { print_lines(@output); writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTCRS_STACK"); trace("crsctl stat res $resName -p failed with status $rc"); die(dieformat(180, "crsctl stat res $resName -p")); } foreach my $line (@output) { if ($line =~ /ACTION_SCRIPT/) { my $actionScript = (split(/=/, $line))[1]; $asmhome = (split(/bin/, $actionScript))[0]; trace("ASM home = $asmhome"); chop($asmhome); trace("ASM home after chopping: $asmhome"); last; } } if (! $asmhome) { writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTCRS_STACK"); die("Unexpected error in retrieving ASM home for ASM reource '$resName'\n"); } else { writeCkptProperty($ckptName, $resName, $asmhome); } trace("Remove ASM resource: <$resName>"); @output = system_cmd_capture($crsctl, "delete", "resource", $resName, "-f", "-unsupported"); $rc = shift @output; if (0 != $rc) { print_lines(@output); writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTCRS_STACK"); trace("crsctl delete resource $resName failed with status $rc"); die(dieformat(180, "crsctl delete resource $resName")); } } if (! $asmhome) { trace("Reading from CKPT file ..."); $asmhome = trim(getCkptPropertyValue($ckptName, $resName)); } if (! $asmhome) { writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTCRS_STACK"); die("Unexpected error in retrieving ASM home for ASM reource '$resName'\n") } return $asmhome; } sub srvctlRemoveASM { my $resName = $_[0]; my $node_name = $_[1]; my $asmhome = $_[2]; my $ckptName = "ROOTCRS_FORCEUPG_RMASMRES"; my @output; my $run_as_owner = FALSE; trace("Remove OCR keys for ASM resource: <$resName>"); my $old_env = $ENV{'ORACLE_HOME'}; $ENV{'ORACLE_HOME'} = $asmhome; trace("Invoke srvctl from ASM home: $asmhome"); my $rc = srvctl_capture($run_as_owner, \@output, "remove asm -n $node_name -f", $asmhome); if ($old_env) { $ENV{'ORACLE_HOME'} = $old_env; } else { undef $ORACLE_HOME; } if ((0 != $rc) && (2 != $rc)) { writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTCRS_STACK"); print_lines(@output); die(dieformat(180, "srvctl remove asm -n $node_name -f")); } return SUCCESS; } sub asmcaJoinCluster { my $asmca = catfile($CFG->ORA_CRS_HOME, "bin", "asmca"); my @runasmca = ($asmca, '-silent', '-joinCluster', $CFG->params('ASMCA_ARGS')); trace ("Executing as " . $CFG->params('ORACLE_OWNER') . ": @runasmca"); my $status = run_as_user($CFG->params('ORACLE_OWNER'), @runasmca); if (0 != $status) { trace("Failed to upgrade ASM on a joining node; return error: $status"); return FAILED; } my $localNode = $CFG->HOST; srvctl(TRUE, "stop asm -n $localNode") || return FAILED; srvctl(TRUE, "start asm -n $localNode") || return FAILED; return SUCCESS; } sub getClusterName { my $CLUSTER_NAME = $CFG->params('CLUSTER_NAME'); if ($CLUSTER_NAME) { trace("CLUSTER_NAME=$CLUSTER_NAME"); return $CLUSTER_NAME; } my $olsnodes = catfile($CFG->ORA_CRS_HOME, 'bin', 'olsnodes'); my @cmd = ($olsnodes, '-c'); my @out = system_cmd_capture(@cmd); my $rc = shift @out; my $cluster_name; if ($rc == 0) { $cluster_name = $out[0]; } chomp($cluster_name); trace("cluster_name=$cluster_name"); $CFG->params('CLUSTER_NAME', $cluster_name); return $cluster_name; } sub upgsihamodel #------------------------------------------------------------------------------- # Function: call srvctl upgrade model for SIHA # Args : none # Returns : SUCCESS or FAILED #------------------------------------------------------------------------------- { trace("Invoking srvctl upgrade model for SIHA"); my $orasrvm = $CFG->compSRVM; my $ret = $orasrvm->postUpgradeCurrentNode(); return $ret; } =head2 cleanASMFiles Function for removing ASM files during upgrade from 11201/11202 =head3 Parameters None =head3 Returns None =cut sub cleanASMFiles { if ($CFG->platform_family eq 'windows') { return; } # The setasmgid file is present in /opt/oracle/bin in 11.2.0.1 and 11.2.0.2. # It needs to be cleaned only for those versions. my @oldCrsVer = @{$CFG->oldconfig('ORA_CRS_VERSION')}; my $verinfo = join('.', @oldCrsVer); if ((0 != versionComparison($verinfo, "11.2.0.1.0")) && (0 != versionComparison($verinfo, "11.2.0.2.0"))) { trace("Old version not eligible. Skipping clean ..."); return; } trace("Cleaning up ASM files ..."); my $optorcldir = $CFG->params('EXTERNAL_ORACLE'); my $optorclbindir = $CFG->params('EXTERNAL_ORACLE_BIN'); my $ORACLE_OWNER = $CFG->params('ORACLE_OWNER'); my $ORA_DBA_GROUP = $CFG->params('ORA_DBA_GROUP'); if (-d $optorclbindir) { my @res; opendir(DIR, $optorclbindir); my @files = readdir(DIR); close DIR; foreach my $file (@files) { if ($file =~ /^setasmgid/) { s_remove_file("$optorclbindir/$file"); } else { push(@res, $file); } } my $noEmpty = 0; foreach my $file (@res) { if (($file eq '.') || ($file eq '..')) { next; } $noEmpty = 1; } if (0 == $noEmpty) { trace("Removing $optorclbindir"); rmtree($optorclbindir); } } # We don't want to fail the upgrade if the clean fails, so # we just trace it and move forward. if (-d $optorclbindir) { trace("Could not reomve the directory '$optorclbindir'"); } if (-d $optorcldir) { # When $optorcldir is owned by root if (-o "$optorcldir") { trace("Change the owner of $optorcldir to $ORACLE_OWNER"); if (FAILED == s_set_ownergroup($ORACLE_OWNER, $ORA_DBA_GROUP, $optorcldir)) { trace("Could not change ownership on $optorcldir"); } } # Skipping this if $optorcldir/.ssh exists my $sshDir = catfile($optorcldir, '.ssh'); if (! (-e $sshDir)) { trace("Change the permissons of $optorcldir"); if (FAILED == s_set_perms("0775", $optorcldir)) { trace("Could not change permissions on $optorcldir"); } } } } =head2 resetASM Reset cardinality of ASM on the last node =head3 Parameters [0] crs home : CRS Home to use for commands [1] ckptName : checkpoint name (passed during upgrade, not in downgrade) =head3 Returns TRUE or FALSE =cut sub resetASM #------------------------------------------------------------------------------ ## Function: Reset cardinality of ASM ## Args : See above ## Notes : Retrieve the previously stored ASM cardinality value and use it ## to reset ASM cardinality using 'srvctl modify asm -count ' ## on the last node in the cluster ## Check for checkpoint value if a valid checkpoint is passed ## Only issue SRVCTL command if the cardinality value retrieved is ## non zero ## If the retrieved cardinality value is ALL, it means the ## cardinality was ALL to begin with. ##----------------------------------------------------------------------------- { my $crshome = $_[0]; my $ckptName = $_[1]; my $ret = TRUE; my $run_as_owner = FALSE; my $status; my $cardinality; my $cmdstr; # Check if the cardinality of Flex ASM was already reset. If already # completed, then skip the step. if ($ckptName ne "") { if (isCkptPropertyExists ($ckptName, "RESET_ASM_CARDINALITY")) { trace("ASM cardinality has been reset."); return $ret; } } # Retrieve ASM cardinality from global CKPT $cardinality = getCkptPropertyValue("ROOTCRS_SETASMCARD", "CARDINALITY", "-global"); # '-1' will be translated to 'ALL' during first node upgrade while storing # ASM cardinality in global CKPT trace("ASM cardinality is: $cardinality"); # Check if the cardinality retrieved is non zero. If zero, then skip # the SRVCTL command if (($cardinality =~ /^\d+$/) && ($cardinality == 0)) { trace("Reset ASM is a NOOP"); return $ret; } # Set ASM cardinality to the stored value $cmdstr = "modify asm -count $cardinality"; trace("Executing srvctl $cmdstr"); my @output; $status = srvctl_capture($run_as_owner, \@output, $cmdstr, $crshome); trace(" \"srvctl ${cmdstr}\" returned with status ${status}"); trace_lines(@output); if ((0 != $status) && (2 != $status) && (0 == scalar(grep(/PRKF-1348/, @output)))) { print_lines(@output); trace("Failed to reset cardinality of Flex ASM"); $ret = FALSE; return $ret; } trace("Successfully reset cardinality of Flex ASM"); if ($ckptName ne "") { writeCkptProperty ($ckptName, "RESET_ASM_CARDINALITY", TRUE); } return $ret; } =head2 upgrade_asm_credentials Create the credential domains for ASM and Gridhome during upgrade =head3 Parameters None =head3 Returns None =cut sub upgrade_asm_credentials { # Do not create the credentials in OCR when they already exist since the # permissions should not be reset. if (isOldVersionLT121()) { trace("Create the credential domains in OCR for ASM and Gridhome"); createCredDomain('ASM', 'OCR') || die(dieformat(377)); createCredDomain('GRIDHOME', 'OCR') || die(dieformat(380)); } elsif (isOldVersion121()) { # Bug 26654615: For some reason when updrading from 11.2 to 12.1, it fails # in last node and cred domains are not created, then when upgrading to # 12.2, it fails bacause cred are not found. Now we are going to create the # domain even if version is not old (before 12.1). my $crsctl_path = crs_exec_path('crsctl'); my (@crsctl_cmd, @crsctl_out) = (); my $rc = -1; # check if path credentials.domains.root.ASM.Self exists @crsctl_cmd = ($crsctl_path, 'query', 'credmaint', '-path', 'ASM'); @crsctl_out = system_cmd_capture(@crsctl_cmd); $rc = shift @crsctl_out; if (1 == $rc) { trace("ASM root domain should have existed. Creating it anyway"); createCredDomain('ASM', 'OCR') || die(dieformat(377)); createCredDomain('GRIDHOME', 'OCR') || die(dieformat(380)); } } # The gpnp profile is backed up and restored when upgrading from # a) legacy/standard asm # b) Standard cluster with flex asm if (isLegacyASM() || (getClusterConfigMode() eq CLUSTER_MODE_STD)) { trace("Taking a backup of lower version GPnP profile ..."); backupGPNPProfile() or exit(1); } if (isLegacyASM()) { trace("Create ASM credentials when upgrading from legacy ASM"); # Bug20621234: ASM credential domain must be present in OCR # before calling 'kfod' to create ASM credentials create_asm_credentials(); # Bug 24287160: avoid enabling the resources on rerun. my $ckptName = "ROOTCRS_RESOURCEENABLED"; my $wipCkpt = $CFG->wipCkptName; if (! isCkptSuccess($ckptName)) { $CFG->wipCkptName($ckptName); writeCkpt($ckptName, CKPTSTART); if(isOldVersionLT121()) { my $crsctl = crs_exec_path('crsctl'); my @out = system_cmd_capture($crsctl, "modify", "resource", "ora.crsd", "-init", "-attr", '"START_MODE=normal"'); my $status = shift @out; trace("The command $crsctl modify resource ora.crsd -init". "-attr \"START_MODE=normal\" returned wih $status"); if ($status != 0) { print_lines(@out); trace("\"$crsctl modify resource ora.crsd -init -attr\" failed with status $status"); die(dieformat(180, 'crsctl modify resource ora.crsd -init -attr')); } } else { my $crshome = $CFG->ORA_CRS_HOME; trace("Enabling crs resources"); setCRSResourceUseAttr($crshome, 'enable') || die(dieformat(180, 'crsctl set resource use')); } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName($wipCkpt); } # Need to bounce the stack to enable remote ASM (bug 13899801)? if(isOldVersionLT121()) { stopFullStack("force") || die(dieformat(349)); startOhasdOnly() || die(dieformat(117)); startStackFor112Upg(); if (!check_service("crs", 120)) { die(dieformat(251)); } } else { stopFullStack("force") || die(dieformat(349)); startFullStack($CFG->ORA_CRS_HOME) || die(dieformat(117)); check_resource_state($CFG->ORA_CRS_HOME) || die(dieformat(117)); } print_info(343); } } =head2 preForceupgChecks Pre-checks for force upgrade =head3 Parameters None =head3 Returns SUCCESS if successful FAILED if failed =cut sub preForceupgChecks { trace("Performing force upgrade pre checks ..."); if (isFirstNodeToUpgrade()) { trace("No need to check on the first node to upgrade"); return SUCCESS; } my $relver = getcrsrelver($CFG->ORA_CRS_HOME); my @activever = get_crs_version($CFG->oldconfig('ORA_CRS_HOME')); # This checks that the cluster has not already been upgraded if (isVersionMatch($relver, join('.', @activever))) { my $ckptstatus = getCkptStatus("ROOTCRS_STACK"); my $lastnodeStatus = getCkptStatus("ROOTCRS_LASTNODE", "-global"); if ($ckptstatus eq CKPTSUC && $lastnodeStatus eq CKPTSUC) { set_bold(); print_error(445, $relver); reset_bold(); exit(0); } } my $softver = getcursoftversion($CFG->HOST); # This checks that the current node has been upgraded if (!(isVersionMatch($relver, $softver))) { print_error(446); return FAILED; } my @upgradedNodes; my @notupgradedNodes; my %nodeStatus = getNodeStatus($CFG->ORA_CRS_HOME); # get the list of configured nodes my @clunodes = keys %nodeStatus; foreach my $clunode (@clunodes) { if (1 == $nodeStatus{$clunode}) # active node { push(@upgradedNodes, $clunode); } else { $softver = getcursoftversion($clunode); # The inactive node list should only contain un-upgraded nodes if (!isVersionMatch($relver, $softver)) { push(@notupgradedNodes, $clunode); } else { trace("The node $clunode may have been upgraded ". "but is inactive right now"); push(@upgradedNodes, $clunode); } } } trace("Cluster nodes : @clunodes"); trace("Upgraded nodes : @upgradedNodes"); trace("Not upgraded nodes: @notupgradedNodes"); my $forceupg = 1; # We must make sure that the force upgrade is applicable to # all active nodes foreach my $clunode (@upgradedNodes) { if ($CFG->HOST =~ /^$clunode$/i) { next; } $softver = getcursoftversion($clunode); if (!isVersionMatch($relver, $softver)) { print_error(447, $clunode); $forceupg = 0; } } if (0 == $forceupg) { return FAILED; } @force_upgrade_nodes = @upgradedNodes; return SUCCESS; } sub preUpgChecks { if (! $CFG->UPGRADE) { return; } # pull the global CKPT from a reachable node if it does not exit my $global_ckpt_file = catfile ($CFG->params('ORACLE_BASE'), "crsdata", "\@global", "crsconfig", "ckptGridHA_global.xml"); unless (-e $global_ckpt_file) { trace("Golbal checkpoint file is missing, Copying it from a reachable " . "node."); my $rmtNodeList = $CFG->params('NODE_NAME_LIST'); copy_file_from_livenode($global_ckpt_file, $global_ckpt_file, $rmtNodeList) || die(dieformat(571, $global_ckpt_file, $rmtNodeList, $global_ckpt_file)); } if ((! $CFG->JOIN) && (! $CFG->skipUpgCheck)) { my $firstCkptState = getCkptStatus("ROOTCRS_FIRSTNODE", "-global"); my $lastCkptState = getCkptStatus("ROOTCRS_LASTNODE", "-global"); if (($firstCkptState eq CKPTSUC) && ($lastCkptState eq CKPTSUC)) { if ($CFG->FORCE) { trace("Both first-node and last-node operations have been done."); trace("Oracle Clusterware has already been successfully ". "configured; hence exiting ..."); set_bold(); print_info(456); reset_bold(); exit(0); } else { my $localNode = tolower_host(); trace("'JOIN' option should be provided for upgrading local " . "node $localNode, because the clusterware has been " . "successfully upgraded."); die(dieformat(669,$localNode)); } } } if (!checkOldCrsOwner()) { die(dieformat(362)); } } =head2 get_HAIP_ENABLED_fromOld The function for retrieving HAIP ENABLED attribute from the old stack The function can only be called when the stack is up =head3 Parameters None =head3 Returns Current HAIP ENABLED: 1 if enabled =cut sub get_HAIP_ENABLED_fromOld { my $val; my ($pname, $pval); trace("get HAIP from old stack"); if ($CFG->platform_family eq "windows") { # non-op: no HAIP on windows return 0; } else { my $CRSCTLBIN = catfile($CFG->ORA_CRS_HOME, 'bin', 'crsctl'); my $cmd = "$CRSCTLBIN status res ora.cluster_interconnect.haip -p -init"; my @output = system_cmd_capture($cmd); my $rc = shift @output; if (0 == $rc) { my @haipEnabledLine = grep(/ENABLED/, @output); if (scalar(@haipEnabledLine) > 0) { my ($key, $value) = split (/=/, $haipEnabledLine[0]); $key = trim($key); $value = trim($value); trace("split ouput key is $key, value is $value"); return $value; } elsif (scalar(grep(/CRS-2613/, @output)) > 0) { trace("HAIP resource does not exist."); return 0; } else { die(dieformat(246)); } } else { print_lines(@output); trace("$cmd failed with status $rc"); die(dieformat(180, $cmd)); } } } sub upgradeMgmtdb { my $savedPfile = catfile($CFG->OLD_CRS_HOME, 'dbs', 'init-dropmgmtdbSAVED.ora'); trace("remove the saved pfile $savedPfile for mgmtdb"); s_remove_file($savedPfile); if (! isOldVersionLT121()) { trace("The mgmtdb is configured, hence removing it ..."); if ( NODE_ROLE_HUB eq $CFG->oldconfig('NODE_CONFIG_ROLE') ) { # Create new pfile on each hub node during upgrade. trace("Creating a new pfile for dropping mgmtdb."); my $pfile = createMgmtdbPfile($CFG->OLD_CRS_HOME, getMgmtdbSPfile($CFG->OLD_CRS_HOME)); } if (isLastNodeToUpgrade()) { trace("Last node: setting the attribute ORACLE_HOME of mgmtdb " . "to the old home"); my $old_gihome = $CFG->OLD_CRS_HOME; my $crsctl = crs_exec_path('crsctl'); my @cmd = ($crsctl, 'modify', 'resource', 'ora.mgmtdb', '-attr', "\"ORACLE_HOME=$old_gihome\"", '-unsupported'); my @output = system_cmd_capture(@cmd); my $rc = shift(@output); if (0 != $rc) { print_lines(@output); trace(join(' ', @cmd) . " failed with status $rc"); die(dieformat(180, join(' ', @cmd))); } } } } sub disableGimrDb { my $ckptStatus; my $ckptName = "ROOTCRS_DISABLEMGMTDB"; $CFG->wipCkptName($ckptName); if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); trace("'$ckptName' state is $ckptStatus"); if ($ckptStatus eq CKPTSUC) { trace("No need to disable the ora.mgmtdb resource"); $CFG->wipCkptName("ROOTCRS_STACK"); return SUCCESS; } } else { writeCkpt($ckptName, CKPTSTART); } if ((! isOldVersionLT121()) && (isFirstNodeToUpgrade()) && (isMgmtdbConfigured($CFG->OLD_CRS_HOME)) && (isMgmtDbEnabled($CFG->OLD_CRS_HOME))) { if (($CFG->platform_family eq 'windows') && (! isMgmtdbRunningOnLocalNode($CFG->OLD_CRS_HOME))) { trace("Relocate the mgmtdb on installer node"); my $host = tolower_host(); srvctl(TRUE, "relocate mgmtdb -n $host", $CFG->OLD_CRS_HOME) || exit(1); } trace("First node: disable ora.mgmtdb"); set_flag_for_ocr_changed(); my @output; # Do nothing with the output, just prevent it gets printed srvctl_capture(TRUE, \@output, "enable mgmtdb", $CFG->OLD_CRS_HOME); # do not check for the return status since the srvctl enable returns with # incorrect return status in 12.1.0.1. # Without the global enable, the disable would not global disable the resource, # if the resource is enabled on some servers but not globally enabled. srvctl(TRUE, "disable mgmtdb", $CFG->OLD_CRS_HOME) || die(dieformat(501)); if (! isOldVersionLT122()) { # stop the mgmtdb to prevent clients accessing it. It will be upgraded by # dbua invoked by installer after rootupgrade finishes on all nodes srvctl(TRUE, "stop mgmtdb -f", $CFG->OLD_CRS_HOME) || die(dieformat(499)); } } writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName("ROOTCRS_STACK"); return SUCCESS; } sub modifyDropdbFile { if ($CFG->platform_family ne 'unix') { return; } trace("Make OLD_GI_HOME have the proper value in dropdb"); my $oldHome = $CFG->OLD_CRS_HOME; my $dropdb = catfile($CFG->ORA_CRS_HOME, "crs", "install", "dropdb"); my $dropdbNew = catfile($CFG->ORA_CRS_HOME, "crs", "install", "dropdb.new"); my @lines = read_file($dropdb); open(DROPDBNEW, ">${dropdbNew}") or die(dieformat(207, $dropdbNew, $!)); foreach my $line (@lines) { if ($line =~ /OLD_GI_HOME=/) { print DROPDBNEW "OLD_GI_HOME=$oldHome\n"; } else { print DROPDBNEW "$line"; } } close(DROPDBNEW); trace("Copying file $dropdbNew to $dropdb"); copy_file($dropdbNew, $dropdb); trace("Removing file $dropdbNew"); s_remove_file($dropdbNew); } =head2 set_CSSD_loglevel The function for setting log level for CSSD =head3 Parameters [0] Log level set for CSSD [1] CRS home =head3 Returns SUCCESS or FAILED =cut sub set_CSSD_loglevel { my $ll = $_[0]; my $crshome = $_[1]; my $CRSCTL = catfile((($crshome) ? ($crshome) : ($CFG->ORA_CRS_HOME)), 'bin', 'crsctl'); trace("Attempt to set CSSD log level to $ll"); my @cmd = ($CRSCTL, 'set', 'log', 'css', "\"CSSD=$ll\""); my @out = system_cmd_capture(@cmd); my $rc = shift(@out); if (0 != $rc) { trace("Failed to set CSSD log level to $ll"); return FAILED; } return SUCCESS; } sub ha_upgrade_ocr { my $status; my $oraocr = $CFG->compOCR; # check if older version SI CSS exists, by looking in OCRLOC $CFG->restart_css(FALSE); my $ckptName = "ROOTHAS_LOCOCR"; if (isCkptSuccess($ckptName)) { trace("Local-only OCR has been created."); $CFG->wipCkptName("ROOTHAS_STACK"); } else { writeCkpt($ckptName, CKPTSTART); $CFG->wipCkptName($ckptName); # Configure local only OCR and pin the node if (! $oraocr->upgradeCurrentNode()) { trace ("Creating local-only OCR ... failed"); writeCkpt($ckptName, CKPTFAIL); $CFG->wipCkptName("ROOTHAS_STACK"); die(dieformat(156)); } else { trace ("Creating local-only OCR ... succeeded"); writeCkpt($ckptName, CKPTSUC); $CFG->wipCkptName("ROOTHAS_STACK"); } } # Initialize the SCR settings. s_init_scr (); } # This function needs to be called during upgrade every time # the OCR contents change. sub set_flag_for_ocr_changed { my $ckptName = "ROOTCRS_OCRSTATUS"; if (! isCkptSuccess($ckptName)) { trace("Write local CKPT that OCR has been changed"); writeCkptProperty($ckptName, "OCR_CHANGED", "true"); writeCkpt($ckptName, CKPTSUC); print_info(486); print_info(541); print_info(542); print_info(615); } } sub get_siha_oldconfig_info { my $self = shift; my $siha_home; my $ckptName = "ROOTHAS_OLDHOMEINFO"; my $ckptStatus; trace("Retrieving old SIHA configuration data"); if (isCkptexist($ckptName)) { $ckptStatus = getCkptStatus($ckptName); } else { writeCkpt($ckptName, CKPTSTART); } # get old OCRCONFIG my $ocrcfg_loc; if ($ckptStatus eq CKPTSUC) { $ocrcfg_loc = getCkptPropertyValue($ckptName, "OCRCONFIG"); } else { $ocrcfg_loc = s_get_config_key("ocr", "ocrconfig_loc"); } trace("Old OCRCONFIG = $ocrcfg_loc"); $CFG->oldconfig('OCRCONFIG', $ocrcfg_loc); # get old home my $oldcrshome; if ($ckptStatus eq CKPTSUC) { $oldcrshome = getCkptPropertyValue($ckptName, "OLD_CRS_HOME"); } else { $oldcrshome = s_get_olr_file ("crs_home"); } trace("Old crs home = $oldcrshome"); $CFG->OLD_CRS_HOME($oldcrshome); $CFG->oldconfig('ORA_CRS_HOME', $oldcrshome); # get older version SIHA home path my @oldCrsVer; my $oldcrsver; if ($ckptStatus eq CKPTSUC) { $oldcrsver = getCkptPropertyValue($ckptName, "OLD_CRS_VERSION"); @oldCrsVer = split(/\./, $oldcrsver); } else { if ($ocrcfg_loc =~ m/(.+).cdata.localhost.local.ocr/) { $siha_home = $1; if ($siha_home) { trace ("local_only config exists"); #get old releaseversion of SIHA/SI-CSS. @oldCrsVer = get_has_version($siha_home); $oldcrsver = join('.', @oldCrsVer); } else { print_error(181); } } else { print_error(7); } } trace("old version info retrieved is: $oldcrsver"); $CFG->oldconfig('ORA_CRS_VERSION', \@oldCrsVer); if ($ckptStatus ne CKPTSUC) { trace("Write old configuation data into the checkpoint file"); writeCkptProperty($ckptName, "OCRCONFIG", $ocrcfg_loc); writeCkptProperty($ckptName, "OLD_CRS_HOME", $oldcrshome); writeCkptProperty($ckptName, "OLD_CRS_VERSION", $oldcrsver); writeCkpt($ckptName, CKPTSUC); } # remove the lower version checkpoints after the old crs home is retrieved # from the old stack. This is needed so that a subsequent downgrade does not # use the old ckpt file. remove_lower_version_ckpts(); if (! check_service ("ohasd", 2)) { stopOracleRestart() || die(dieformat(348)); startOhasdOnlyInSIHA() || die(dieformat(318)); } } sub removeDiagsnap { my $resource = 'diagsnap'; if ( crsdResource( 'status', $resource ) ) { trace("Removing registered resource - $resource"); crsdResource( 'delete', $resource ); } return SUCCESS; } sub get_cluster_info #------------------------------------------------------------------------------- # Function: Get the list of nodes and their roles on the first node to upgrade. # Store it in the global ckpt file so that it can be used for # downgrade. # Args : none #------------------------------------------------------------------------------- { # get the nodes and their roles among cluster if (isOldVersionLT121()) { # Old version lower than 12.1 has no leaf nodes, # no need to get the list of nodes and roles. return; } my $node_roles; my $globalCkptName = "ROOTCRS_CLUSTERINFO"; my $globalCkptPrpt_nodeRoles = "CLUSTERNODE_ROLES"; my $globalCkptStatus = CKPTFAIL; if (isCkptexist($globalCkptName, "-global")) { $globalCkptStatus = getCkptStatus($globalCkptName, "-global"); } else { writeGlobalCkpt($globalCkptName, CKPTSTART, "-transferfile"); $globalCkptStatus = CKPTSTART; } if ($globalCkptStatus eq CKPTSUC) { $node_roles = trim(getCkptPropertyValue($globalCkptName, $globalCkptPrpt_nodeRoles, "-global")); trace("The cluster nodes and roles from ckpt property: $node_roles"); } else { # 'olsnodes -a' lists active and inactive(down recently) hub nodes # and active leaf nodes. my ($rca, @olsnodes_a_out) = get_olsnodes_info($CFG->ORA_CRS_HOME, '-a'); # 'olsnodes -f' lists active and inactive (down recently) leaf nodes. my ($rcf, @olsnodes_f_out) = get_olsnodes_info($CFG->ORA_CRS_HOME, '-f'); if ($rca != 0 || $rcf != 0) { writeGlobalCkpt($globalCkptName, CKPTFAIL, "-transferfile"); die(dieformat(174)); } else { # output of 'olsnodes -a' would be like: # rwsad11 Hub # rwsad12 Leaf # rwsad13 None my @tempArray; foreach my $line (@olsnodes_a_out) { chomp($line); $line =~ s/\s+/:/; # 'None' indicates the node is a recently down hub node. $line =~ s/None/Hub/i; push @tempArray, $line; } # output of 'olsnodes -f' only contains node names like: # adc01axz # adc01ayo foreach my $line (@olsnodes_f_out) { chomp($line); # value from 'olsnodes -f' should be stored only when it's not in the # array list. unless (@tempArray ~~ /^$line\:/) { $line = $line.":Leaf"; push @tempArray, $line; } } $node_roles = join(',', @tempArray); trace("The nodes and their roles: $node_roles"); writeCkptProperty($globalCkptName, $globalCkptPrpt_nodeRoles, $node_roles, "-global", "-transferfile"); writeGlobalCkpt($globalCkptName, CKPTSUC, "-transferfile"); } } } # Function: Remove global ckpt file from lower version install or upgrade. # It's called only during upgrade. sub create_starting_global_ckpt_entry { my $ckptName = "ROOTCRS_FIRSTNODE"; my $crsrelver = getcrsrelver1(); if (!isCkptexist($ckptName, "-global")) { trace("Grid Infrastructure upgrade has started."); writeGlobalCkpt($ckptName, CKPTSTART, "-transferfile"); writeCkptProperty($ckptName, "VERSION", $crsrelver, "-global", "-transferfile"); } else { if (!isCkptPropertyExists($ckptName, "VERSION", "-global")) { trace("removing older global checkpoints from previous install"); remove_global_checkpoints(); writeGlobalCkpt($ckptName, CKPTSTART, "-transferfile"); writeCkptProperty($ckptName, "VERSION", $crsrelver, "-global", "-transferfile"); } else { my $ckptcrsver = getCkptPropertyValue($ckptName, "VERSION", "-global"); if (!isVersionMatch($ckptcrsver, $crsrelver)) { trace("removing older global checkpoints from previous upgrade"); remove_global_checkpoints(); writeGlobalCkpt($ckptName, CKPTSTART, "-transferfile"); writeCkptProperty($ckptName, "VERSION", $crsrelver, "-global", "-transferfile"); } else { # version match, it's re-run on the first node or upgrade on the other node. trace("no need to remove global checkpoints"); } } } } #------------------------------------------------------------------------------- # Function: Disable and stop GIMR clients running from the old home. # This is called in the context of upgrading the GIMR database for # upgrades from 12201 onwards. It's done on the first node before # the old stack is stopped. # Args : None # Returns : TRUE or FALSE #------------------------------------------------------------------------------- sub disableGIMRClients { my $crsHome = $CFG->OLD_CRS_HOME; trace("Attempt to disable and stop GIMR clients"); if (isFirstNodeToUpgrade()) { my $orachm = $CFG->compCHM; if (isCHAConfigured()) { my $rc; my @output; my $orachm = $CFG->compCHM; $orachm->disable_CHA() || return FALSE; srvctl(TRUE, "stop cha -f", $crsHome) || return FALSE; } if (check_qosmserver_supported()) { (disable_qosmserver() && stop_qosmserver()) || return FALSE; } } return TRUE; } # Function: remove the lower version checkpoint file during SIHA upgrade sub remove_lower_version_ckpts { # retrieve the old oracle base my $old_base; my @old_params = read_file(catfile($CFG->OLD_CRS_HOME, 'crs', 'install', 'crsconfig_params')); foreach my $line (@old_params) { if ($line =~ /ORACLE_BASE=(.+)/) { $old_base = $1; } } return if ($old_base eq $CFG->params('ORACLE_BASE')); my $host = tolower_host(); my $ckpt_filename = "ckptGridHA_".$host.".xml"; my $old_ckpt_file = catfile($old_base, 'crsdata', $host, 'crsconfig', $ckpt_filename); trace("Removing the lower version checkpoint file $old_ckpt_file"); s_remove_file ($old_ckpt_file); } 1;