[6652] | 1 | # -*- coding: utf-8 -*- |
---|
| 2 | ## =========================================================================== |
---|
| 3 | ## |
---|
| 4 | ## This software is governed by the CeCILL license under French law and |
---|
| 5 | ## abiding by the rules of distribution of free software. You can use, |
---|
| 6 | ## modify and/ or redistribute the software under the terms of the CeCILL |
---|
| 7 | ## license as circulated by CEA, CNRS and INRIA at the following URL |
---|
| 8 | ## "http://www.cecill.info". |
---|
| 9 | ## |
---|
| 10 | ## Warning, to install, configure, run, use any of Olivier Marti's |
---|
| 11 | ## software or to read the associated documentation you'll need at least |
---|
| 12 | ## one (1) brain in a reasonably working order. Lack of this implement |
---|
| 13 | ## will void any warranties (either express or implied). |
---|
| 14 | ## O. Marti assumes no responsability for errors, omissions, |
---|
| 15 | ## data loss, or any other consequences caused directly or indirectly by |
---|
| 16 | ## the usage of his software by incorrectly or partially configured |
---|
| 17 | ## personal. |
---|
| 18 | ## |
---|
| 19 | ## =========================================================================== |
---|
| 20 | ''' |
---|
| 21 | Defines libIGCM directories, depending of the computer |
---|
| 22 | |
---|
| 23 | olivier.marti@lsce.ipsl.fr |
---|
| 24 | ''' |
---|
| 25 | |
---|
| 26 | import sys, os, subprocess, configparser, re, types |
---|
| 27 | from pathlib import Path |
---|
| 28 | |
---|
| 29 | # Where do we run ? |
---|
| 30 | SysName, NodeName, Release, Version, Machine = os.uname () |
---|
| 31 | |
---|
| 32 | def unDefined (char) : |
---|
| 33 | '''Returns True if a variable is not defined, ot if it's set to None''' |
---|
| 34 | if char in globals () : |
---|
| 35 | if globals ()[char] == None or globals ()[char] == 'None': unDefined = True |
---|
| 36 | else : unDefined = False |
---|
| 37 | else : unDefined = True |
---|
| 38 | return unDefined |
---|
| 39 | |
---|
| 40 | def Mach ( Long=False) : |
---|
| 41 | ''' |
---|
| 42 | Find the computer we are on |
---|
| 43 | |
---|
| 44 | On Irene, Mach returns Irene, Irene-Next, Rome or Rome-Prev if Long==True |
---|
| 45 | ''' |
---|
| 46 | |
---|
| 47 | Mach = 'unknown' |
---|
| 48 | |
---|
| 49 | if SysName == 'Darwin' and 'lsce5138' in NodeName : Mach = 'Spip' |
---|
| 50 | if 'obelix' in NodeName : Mach = 'Obelix' |
---|
| 51 | if 'forge' in NodeName : Mach = 'Forge' |
---|
| 52 | if 'ciclad' in NodeName : Mach = 'Ciclad' |
---|
| 53 | if 'climserv' in NodeName : Mach = 'SpiritX' |
---|
| 54 | if 'spirit' in NodeName : Mach = 'SpiritJ' |
---|
| 55 | if 'spiritj' in NodeName : Mach = 'SpiritJ' |
---|
| 56 | if 'spiritx' in NodeName : Mach = 'SpiritX' |
---|
| 57 | if 'irene' in NodeName : Mach = 'Irene' |
---|
| 58 | if 'jean-zay' in NodeName : Mach = 'Jean-Zay' |
---|
| 59 | |
---|
| 60 | if Long : |
---|
| 61 | MachFull = Mach |
---|
| 62 | |
---|
| 63 | if Mach == 'Irene' : |
---|
| 64 | CPU = subprocess.getoutput ('lscpu') |
---|
| 65 | ccc_os = subprocess.getoutput ('ccc_os') |
---|
| 66 | |
---|
| 67 | if "Intel(R) Xeon(R) Platinum" in CPU : |
---|
| 68 | if "Atos_7__x86_64" in ccc_os : MachFull = 'Irene-Prev' |
---|
| 69 | if "Rhel_8__x86_64" in ccc_os : MachFull = 'Irene' |
---|
| 70 | |
---|
| 71 | if "AMD" in CPU : |
---|
| 72 | if "Atos_7__x86_64" in ccc_os : MachFull = 'Rome-Prev' |
---|
| 73 | if "Rhel_8__x86_64" in ccc_os : MachFull = 'Rome' |
---|
| 74 | |
---|
| 75 | Mach = MachFull |
---|
| 76 | |
---|
| 77 | return Mach |
---|
| 78 | |
---|
| 79 | def config ( JobName=None , TagName=None , SpaceName=None, ExperimentName=None, |
---|
| 80 | LongName=None, ModelName=None, ShortName=None, |
---|
| 81 | Source=None , Host=None , ConfigCard=None, User=None, Group=None, |
---|
| 82 | TGCC_User='p86mart', TGCC_Group='gen12006', IDRIS_User='rces009', IDRIS_Group='ces', |
---|
| 83 | ARCHIVE=None, SCRATCHDIR=None, STORAGE=None, R_IN=None, R_OUT=None, R_FIG=None, L_EXP=None, |
---|
| 84 | R_SAVE=None , R_FIGR=None, R_BUF=None , R_BUFR=None, R_BUF_KSH=None, REBUILD_DIR=None, POST_DIR=None, |
---|
| 85 | ThreddsPrefix=None, R_GRAF=None, DB=None, |
---|
| 86 | IGCM_OUT=None, IGCM_OUT_name=None, rebuild=None, TmpDir=None, |
---|
| 87 | Out='dict') : |
---|
| 88 | ''' |
---|
| 89 | Defines the libIGCM directories - Returns a dictionnary or a namespace |
---|
| 90 | |
---|
| 91 | Source : for Spip |
---|
| 92 | Possibilities : |
---|
| 93 | Local : local (~/Data/IGCMG/...) |
---|
| 94 | TGCC_sshfs : TGCC disks mounted via sshfs |
---|
| 95 | TGCC_thredds : thredds TGCC via IPSL ('https://thredds-su.ipsl.fr/thredds/dodsC/tgcc_thredds/store/...) |
---|
| 96 | |
---|
| 97 | Exemple of use : |
---|
| 98 | libIGCM = libIGCM_sys.config ( .... ) |
---|
| 99 | globals().update ( libIGCM ) |
---|
| 100 | |
---|
| 101 | ''' |
---|
| 102 | #if not IGCM_OUT_name : |
---|
| 103 | # if Source == 'TGCC_thredds' : IGCM_OUT_name = '' |
---|
| 104 | # else : IGCM_OUT_name = 'IGCM_OUT' |
---|
| 105 | |
---|
| 106 | if not Host : Host = Mach (Long=False) |
---|
| 107 | |
---|
| 108 | LocalUser = os.environ ['USER'] |
---|
| 109 | |
---|
| 110 | # =========================================================================================== |
---|
| 111 | # Reads config.card if available |
---|
| 112 | if ConfigCard : |
---|
| 113 | # Text existence of ConfigCard |
---|
| 114 | if not os.path.exists (ConfigCard ) : |
---|
| 115 | raise FileExistsError ( f"File not found : {ConfigCard = }" ) |
---|
| 116 | |
---|
| 117 | ## Creates parser for reading .ini input file |
---|
| 118 | MyReader = configparser.ConfigParser (interpolation=configparser.ExtendedInterpolation() ) |
---|
| 119 | MyReader.optionxform = str # To keep capitals |
---|
| 120 | |
---|
| 121 | MyReader.read (ConfigCard) |
---|
| 122 | |
---|
| 123 | JobName = MyReader['UserChoices']['JobName'] |
---|
| 124 | #----- Short Name of Experiment |
---|
| 125 | ExperimentName = MyReader['UserChoices']['ExperimentName'] |
---|
| 126 | #----- DEVT TEST PROD |
---|
| 127 | SpaceName = MyReader['UserChoices']['SpaceName'] |
---|
| 128 | LongName = MyReader['UserChoices']['LongName'] |
---|
| 129 | ModelName = MyReader['UserChoices']['ModelName'] |
---|
| 130 | TagName = MyReader['UserChoices']['TagName'] |
---|
| 131 | |
---|
| 132 | ### =========================================================================================== |
---|
| 133 | ## Part specific to access by OpenDAP/Thredds server |
---|
| 134 | |
---|
| 135 | # =========================================================================================== |
---|
| 136 | if Source == 'TGCC_thredds' : |
---|
| 137 | if not User and TGCC_User : User = TGCC_User |
---|
| 138 | if not Group and TGCC_Group : Group = TGCC_Group |
---|
| 139 | |
---|
| 140 | if not ThreddsPrefix : ThreddsPrefix = 'https://thredds-su.ipsl.fr/thredds/dodsC/tgcc_thredds' |
---|
| 141 | |
---|
| 142 | if not ARCHIVE : ARCHIVE = f'{ThreddsPrefix}/store/{TGCC_User}' |
---|
| 143 | if not R_FIG : R_FIG = f'{ThreddsPrefix}/work/{TGCC_User}' |
---|
| 144 | if not R_IN : R_IN = f'{ThreddsPrefix}/work/igcmg/IGCM' |
---|
| 145 | if not R_GRAF : R_GRAF = f'{ThreddsPrefix}/work/p86mart/GRAF/DATA' |
---|
| 146 | |
---|
| 147 | # =========================================================================================== |
---|
| 148 | if Source == 'IDRIS_thredds' : |
---|
| 149 | if not User and IDRIS_User : User = IDRIS_User |
---|
| 150 | if not Group and IDRIS_Group : Group = IDRIS_Group |
---|
| 151 | |
---|
| 152 | if not ThreddsPrefix : ThreddsPrefix = 'https://thredds-su.ipsl.fr/thredds/catalog/idris_thredds' |
---|
| 153 | |
---|
| 154 | if not ARCHIVE : ARCHIVE = f'{ThreddsPrefix}/store/{IDRIS_User}' |
---|
| 155 | if not R_FIG : R_FIG = f'{ThreddsPrefix}/work/{IDRIS_User}' |
---|
| 156 | if not R_IN : R_IN = f'{ThreddsPrefix}/work/igcmg/IGCM' |
---|
| 157 | if not R_GRAF : R_GRAF = f'https://thredds-su.ipsl.fr/thredds/dodsC/tgcc_thredds/work/p86mart/GRAF/DATA' |
---|
| 158 | |
---|
| 159 | ### =========================================================================================== |
---|
| 160 | ## Machine dependant part |
---|
| 161 | |
---|
| 162 | # =========================================================================================== |
---|
| 163 | if Host == 'Spip' : |
---|
| 164 | if not User : User = LocalUser |
---|
| 165 | IGCM_OUT_name = 'IGCM_OUT' |
---|
| 166 | if not ARCHIVE : ARCHIVE = os.path.join ( Path.home (), 'Data' ) |
---|
| 167 | if not SCRATCHDIR : SCRATCHDIR = os.path.join ( Path.home (), 'Scratch' ) |
---|
| 168 | if not R_BUF : R_BUF = os.path.join ( Path.home (), 'Scratch' ) |
---|
| 169 | if not R_FIG : R_FIG = os.path.join ( Path.home (), 'Data' ) |
---|
| 170 | |
---|
| 171 | if not STORAGE : STORAGE = ARCHIVE |
---|
| 172 | if not R_IN : R_IN = os.path.join ( '/', 'Users', 'marti', 'Data', 'IGCM' ) |
---|
| 173 | if not R_GRAF : R_GRAF = os.path.join ( '/', 'Users', 'marti', 'GRAF', 'DATA' ) |
---|
| 174 | if not DB : DB = os.path.join ( '/', 'Users', 'marti', 'GRAF', 'DB' ) |
---|
| 175 | if not TmpDir : TmpDir = os.path.join ( '/', 'Users', LocalUser, 'Scratch' ) |
---|
| 176 | |
---|
| 177 | # =========================================================================================== |
---|
| 178 | if ( 'Irene' in Host ) or ( 'Rome' in Host ) : |
---|
| 179 | LocalHome = subprocess.getoutput ( f'ccc_home --ccchome' ) |
---|
| 180 | LocalGroup = os.path.basename ( os.path.dirname (LocalHome)) |
---|
| 181 | if not User or User == 'marti' : |
---|
| 182 | if not TGCC_User : User = LocalUser |
---|
| 183 | else : User = TGCC_User |
---|
| 184 | |
---|
| 185 | if not Group : |
---|
| 186 | if TGCC_Group : Group = TGCC_Group |
---|
| 187 | else : Group = LocalGroup |
---|
| 188 | |
---|
| 189 | IGCM_OUT_name = 'IGCM_OUT' |
---|
| 190 | |
---|
| 191 | if not R_IN : R_IN = os.path.join ( subprocess.getoutput ( f'ccc_home --cccwork -d igcmg -u igcmg' ), 'IGCM') |
---|
| 192 | if not ARCHIVE : ARCHIVE = subprocess.getoutput ( f'ccc_home --cccstore -u {User} -d {Group}' ) |
---|
| 193 | if not STORAGE : STORAGE = subprocess.getoutput ( f'ccc_home --cccwork -u {User} -d {Group}' ) |
---|
| 194 | if not SCRATCHDIR : SCRATCHDIR = subprocess.getoutput ( f'ccc_home --cccscratch -u {User} -d {Group}' ) |
---|
| 195 | if not R_BUF : R_BUF = subprocess.getoutput ( f'ccc_home --cccscratch -u {User} -d {Group}' ) |
---|
| 196 | if not R_FIG : R_FIG = subprocess.getoutput ( f'ccc_home --cccwork -u {User} -d {Group}' ) |
---|
| 197 | if not R_GRAF : R_GRAF = os.path.join ( subprocess.getoutput ( f'ccc_home --cccwork -d drf -u p86mart'), 'GRAF', 'DATA' ) |
---|
| 198 | if not DB : DB = os.path.join ( subprocess.getoutput ( f'ccc_home --cccwork -d igcmg -u igcmg'), 'database' ) |
---|
| 199 | if not rebuild : |
---|
| 200 | rebuild = os.path.join ( subprocess.getoutput ( f'ccc_home --ccchome -d igcmg -u igcmg' ), 'Tools', Machine, 'rebuild_nemo', 'bin', 'rebuild_nemo' ) |
---|
| 201 | if not TmpDir : TmpDir = subprocess.getoutput ( f'ccc_home --cccscratch' ) |
---|
| 202 | |
---|
| 203 | # =========================================================================================== |
---|
| 204 | if Host == 'SpiritJ' : |
---|
| 205 | if not User : |
---|
| 206 | if TGCC_User : User = TGCC_User |
---|
| 207 | else : User = LocalUser |
---|
| 208 | if not ARCHIVE : ARCHIVE = os.path.join ( '/', 'thredds' , 'tgcc', 'store', User ) |
---|
| 209 | if not STORAGE : STORAGE = os.path.join ( '/', 'thredds' , 'tgcc', 'work' , User ) |
---|
| 210 | #if not SCRATCHDIR : SCRATCHDIR = os.path.join ( '/', 'thredds' , 'tgcc', 'store', User ) |
---|
| 211 | if not R_IN : R_IN = os.path.join ( '/', 'projsu', 'igcmg', 'IGCM' ) |
---|
| 212 | #if not R_GRAF : R_GRAF = os.path.join ('/', 'data', 'omamce', 'GRAF', 'DATA' ) |
---|
| 213 | if not R_GRAF : R_GRAF = os.path.join ( '/', 'thredds' , 'tgcc', 'work' , 'p86mart', 'GRAF', 'DATA' ) |
---|
| 214 | if not DB : DB = os.path.join ( '/', 'data', 'igcmg', 'database' ) |
---|
| 215 | if not TmpDir : TmpDir = os.path.join ( '/', 'data', LocalUser ) |
---|
| 216 | |
---|
| 217 | # =========================================================================================== |
---|
| 218 | if Host == 'SpiritX' : |
---|
| 219 | if not User : |
---|
| 220 | if TGCC_User : User = TGCC_User |
---|
| 221 | else : User = os.environ ['USER'] |
---|
| 222 | if not ARCHIVE : ARCHIVE = os.path.join ( '/', 'thredds' , 'tgcc', 'store', User ) |
---|
| 223 | if not STORAGE : STORAGE = os.path.join ( '/', 'thredds' , 'tgcc', 'work' , User ) |
---|
| 224 | #if not SCRATCHDIR : SCRATCHDIR = os.path.join ( '/', 'thredds' , 'tgcc', 'store', User ) |
---|
| 225 | if not R_IN : R_IN = os.path.join ( '/', 'projsu', 'igcmg', 'IGCM' ) |
---|
| 226 | #if not R_GRAF : R_GRAF = os.path.join ('/', 'data', 'omamce', 'GRAF', 'DATA' ) |
---|
| 227 | if not R_GRAF : R_GRAF = os.path.join ( '/', 'thredds' , 'tgcc', 'work' , 'p86mart', 'GRAF', 'DATA' ) |
---|
| 228 | if not DB : DB = os.path.join ( '/', 'data', 'igcmg', 'database' ) |
---|
| 229 | |
---|
| 230 | # =========================================================================================== |
---|
| 231 | if Host == 'Jean-Zay' : |
---|
| 232 | if not User : User = os.environ ['USER'] |
---|
| 233 | LocalGroup = os.path.basename ( os.path.dirname ( Path.home () )) |
---|
| 234 | if not Group : Group = LocalGroup |
---|
| 235 | |
---|
| 236 | if not ARCHIVE : ARCHIVE = os.path.join ( '/', 'gpfsstore' , 'rech', Group, User ) |
---|
| 237 | if not STORAGE : STORAGE = os.path.join ( '/', 'gpfswork' , 'rech', Group, User ) |
---|
| 238 | if not SCRATCHDIR : SCRATCHDIR = os.path.join ( '/', 'gpfsscratch', 'rech', Group, User ) |
---|
| 239 | if not R_FIG : R_FIG = os.path.join ( '/', 'cccwork' , 'rech', Group, User ) |
---|
| 240 | if not R_BUF : R_BUF = os.path.join ( '/', 'gpfsscratch', 'rech', Group, User ) |
---|
| 241 | if not R_IN : R_IN = os.path.join ( '/', 'gpfswork' , 'rech', 'psl', 'commun', 'IGCM' ) |
---|
| 242 | if not R_GRAF : R_GRAF = os.path.join ( '/', 'gpfswork' , 'rech', Group, User, 'GRAF', 'DATA' ) |
---|
| 243 | if not DB : DB = os.path.join ( '/', 'gpfswork' , 'rech', 'psl', 'commun', 'database' ) |
---|
| 244 | if not rebuild : |
---|
| 245 | rebuild = os.path.join ( '/', 'gpfswork', 'rech', 'psl', 'commun', 'Tools', 'rebuild', 'modipsl_IOIPSL_PLUS_v2_2_4', 'bin', 'rebuild' ) |
---|
| 246 | if not TmpDir : TmpDir = os.path.join ( '/', 'gpfsscratch', 'rech', os.path.basename ( os.path.dirname ( Path.home () )), LocalUser ) |
---|
| 247 | |
---|
| 248 | ### =========================================================================================== |
---|
| 249 | ### The construction of the following variables is not machine dependant |
---|
| 250 | ### =========================================================================================== |
---|
| 251 | if SpaceName == 'TEST' : |
---|
| 252 | if SCRATCHDIR and not R_OUT : SCRATCHDIR |
---|
| 253 | if SCRATCHDIR and not R_FIG : SCRATCHDIR |
---|
| 254 | else : |
---|
| 255 | if ARCHIVE and not R_OUT : R_OUT = ARCHIVE |
---|
| 256 | if STORAGE and not R_FIG : R_FIG = STORAGE |
---|
| 257 | if IGCM_OUT_name : |
---|
| 258 | R_OUT = os.path.join ( R_OUT, IGCM_OUT_name ) |
---|
| 259 | R_FIG = os.path.join ( R_FIG, IGCM_OUT_name ) |
---|
| 260 | |
---|
| 261 | if SCRATCHDIR and not R_BUF : R_BUF = os.path.join ( SCRATCHDIR, IGCM_OUT_name ) |
---|
| 262 | if not IGCM_OUT : IGCM_OUT = R_OUT |
---|
| 263 | |
---|
| 264 | if TagName and SpaceName and ExperimentName and JobName : |
---|
| 265 | L_EXP = os.path.join ( TagName, SpaceName, ExperimentName, JobName ) |
---|
| 266 | |
---|
| 267 | if R_OUT and not R_SAVE : R_SAVE = os.path.join ( R_OUT , L_EXP ) |
---|
| 268 | if IGCM_OUT_name : |
---|
| 269 | if STORAGE and not R_FIGR : R_FIGR = os.path.join ( STORAGE, IGCM_OUT_name, L_EXP ) |
---|
| 270 | else : |
---|
| 271 | if STORAGE and not R_FIGR : R_FIGR = os.path.join ( STORAGE, L_EXP ) |
---|
| 272 | if R_BUF and not R_BUFR : R_BUFR = os.path.join ( R_BUF , L_EXP ) |
---|
| 273 | if R_BUFR and not R_BUF_KSH : R_BUF_KSH = os.path.join ( R_BUFR , 'Out' ) |
---|
| 274 | if R_BUF and not REBUILD_DIR : REBUILD_DIR = os.path.join ( R_BUF , L_EXP, 'REBUILD' ) |
---|
| 275 | if R_BUF and not POST_DIR : POST_DIR = os.path.join ( R_BUF , L_EXP, 'Out' ) |
---|
| 276 | |
---|
| 277 | ### =========================================================================================== |
---|
| 278 | ## Builds the final dictionnary |
---|
| 279 | |
---|
| 280 | libIGCM = { 'TagName' : TagName , 'SpaceName' : SpaceName, 'ExperimentName': ExperimentName, 'JobName':JobName, |
---|
| 281 | 'LongName': LongName, 'ModelName' : ModelName, 'ShortName' : ShortName , 'ConfigCard' : ConfigCard, |
---|
| 282 | 'ARCHIVE' : ARCHIVE , 'STORAGE' : STORAGE , 'SCRATCHDIR' : SCRATCHDIR , |
---|
| 283 | 'R_OUT' : R_OUT , 'R_BUF' : R_BUFR , 'R_GRAF' : R_GRAF , 'DB' : DB, |
---|
| 284 | 'IGCM_OUT': IGCM_OUT, 'R_SAVE' : R_SAVE , 'R_FIGR' : R_FIGR , 'R_BUFR': R_BUFR, 'REBUILD_DIR': REBUILD_DIR, |
---|
| 285 | 'POST_DIR': POST_DIR, 'R_IN' : R_IN , 'Mach' : Host , 'Source': Source, |
---|
| 286 | 'User' : User , 'Group' : Group , 'IGCM_OUT_name' : IGCM_OUT_name, 'rebuild':rebuild, 'TmpDir':TmpDir, |
---|
| 287 | 'TGCC_User':TGCC_User, 'TGCC_Group':TGCC_Group, 'Thredds' : ThreddsPrefix, |
---|
| 288 | 'IDRIS_User':IDRIS_User, 'IDRIS_Group':IDRIS_Group } |
---|
| 289 | |
---|
| 290 | if Out in ['namespace', 'space', 'name', 'Space', 'Name', 'Names', 'NameSpace'] : |
---|
| 291 | libIGCM = types.SimpleNamespace (**libIGCM) |
---|
| 292 | |
---|
| 293 | return libIGCM |
---|
| 294 | |
---|