Feel free to join the next Helmholtz Hacky Hour #26 on Wednesday, April 21, 2021 from 2PM to 3PM!

Commit fe0ab9f2 authored by Friedrich Boeing's avatar Friedrich Boeing

updated launch script for new version namelist, added script to loop over variables

parent 058aa0d2
#!/bin/bash
set -e # will stop script when error occurs / return_value != 0
set -x
###############################################
### Script to execute the edk launch script ###
###############################################
# set main paths and variables here
# further settings for job submission etc. in the launch_edk.sh
PWD=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
DWDPATH="/data/klimabuero/GDM/tmp/dwd/" # path containing the station data
EDKSTATIC="/data/klimabuero/GDM/data/EDK/" # path containing the variogram parameters and dem
OUTPATH_MAIN="${PWD}/2020/" # outpath, can be decided!
start_date="2020-01-01"
end_date="2020-05-02"
METEOVARs=("pre" "tavg" "tmin" "tmax")
METEOVARs=("pre")
for var in "${METEOVARs[@]}"; do
bash ${PWD}/launch_edk.sh "${DWDPATH}/${var}/" "${EDKSTATIC}" "${start_date}" "${end_date}" "${var}" "${OUTPATH_MAIN}/${var}"
done
#!/bin/bash
set -e
# set -x
############################################################
##script for copying or creating folders and updating data##
############################################################
# created by Matthias Zink 2011
# adapted by Friedrich Boeing Apr. 2020
############################################################
##script for copying or creating folders and updating data##
#######################mz2011-02-01#########################
# decide to run the job on the frontend (interactive='yes') or on the grid engine ('no')
stationpath=${1}
datapath=${2}
start_date=${3}
end_date=${4}
var=${5} # name of input variable
outpath=${6}
PWD=$(dirname $(readlink -f "${BASH_SOURCE[0]}"))
EDK_PROG_PATH="/data/duerre/GDM_new/local/Progs/edk_latlon/edk"
# decide to run the job on the frontend (interactive='yes') or on the grid engine ('no')
interactive='no'
# define estimated runtime and memeory requirements for the grid engine
rtime='32:00:00' # 540 stations (tavg) ca 7h per year
vmem='4G' #temp 2G pre 4G
rtime='10:00:00'
vmem='12G'
mail='friedrich.boeing@ufz.de'
hmem='true'
OMP_NTHREAD='10'
qsub_manual='false'
# Define directories and presettings
# prefix describing data, in- and output directories
prefix='pre'
# directory of the edk program - executeable
rundir=$(pwd)'/'
# directory of input - data files
inpath="/data/stohyd/data/processed/Germany/DWD/${prefix}/1900-2014_02_23/"
start_year=$(echo ${start_date} | cut -d '-' -f 1)
start_month=$(echo ${start_date} | cut -d '-' -f 2)
start_day=$(echo ${start_date} | cut -d '-' -f 3)
end_year=$(echo ${end_date} | cut -d '-' -f 1)
end_month=$(echo ${end_date} | cut -d '-' -f 2)
end_day=$(echo ${end_date} | cut -d '-' -f 3)
# directory of input - DEM
DEMfile="/data/stohyd/data/processed/Germany/DEM/filled_final1.asc"
#DEMfile="/home/zink/data/edk/dem.test"
dem="${datapath}/dem_hicam_1km_g.nc"
# path and filename of station look-up table
LuT="/data/stohyd/data/processed/Germany/DWD/${prefix}/1900-2014_02_23/StationLuT_${prefix}.txt"
# target directory
outpath="/work/zink/edk_crossloc/vario/" #${i}/"
lut="${stationpath}/StationLuT_${var}.txt"
# cellsize = cellfactor * Gridcellsize_DEM (e.g. 100m)
cellfactor=40
cellfactor=1
# data converting factor --> value(double) = value(integer) * dataconv
dataconv=1.0e-1
# datat format of the output either 'bin' or 'nc'
dataformat="bin"
# flag for interpolation - ".true."=interpolate or ".false."=skip interpolation
edkesti=".FALSE."
# starting year of interpolation
edkyStart=1950
edkmStart=01
edkdStart=01
# ending date of interpolation
edkyEnd=2010
edkmEnd=12
edkdEnd=31
# VARIOGRAM-estimate please see notes below
vario=".TRUE."
vario=".FALSE."
# variogarm type (spherical or exponential)
varType=2
var_type=-9
# number of variogram parameters
varParam=3
var_param=3
# path and filename of first estimation if vario= ".false."
# or calculated variogram parameters if vario= ".true."
#varioFname="/home/zink/source_code/edk/variogram_param_apriori.txt"
varioFname="/data/stohyd/data/processed/Germany/DWD/${prefix}/var_param_de_${prefix}.txt"
vario_fname="${datapath}/var_param_de_${var}.txt"
# parameters for variogram estimation
# width [m] for binning the datapoints of the empirical variogram
binsize=1.0e3
# maximum search distance for data pairs
hMAx=8.0e4
# name of the catchment
catchment="GER"
hmax=8.0e4
# create variables needed for processing
case ${prefix} in
'pre')
case ${var} in
'pre')
datatype="Precipitation"
FlVar=1 # see below
FlMethod=1 # see below
FlMethod=2 # see below
no_data=-9 # nodata value
buffer=1.0e5 # maximal distance of stations taken into consideration for interpolation [m]
var_name='pre'
var_unit='mm d-1'
var_long_name='daily sum of precipitation'
corrNeg='.True.'
distZ='.True.'
;;
'tavg')
'tavg')
datatype="average_Temp"
FlVar=2
FlMethod=1
FlMethod=2
no_data=-999
buffer=1.5e5
var_name='tavg'
var_unit='degC'
var_long_name='mean daily temperature'
corrNeg='.False.'
distZ='.False.'
;;
'tmax')
'tmax')
datatype="max_Temp"
FlVar=2
FlMethod=1
FlMethod=2
no_data=-999
buffer=1.5e5
var_name='tmax'
var_unit='degC'
var_long_name='maximum daily temperature'
corrNeg='.False.'
distZ='.False.'
;;
'tmin')
'tmin')
datatype="min_Temp"
FlVar=2
FlMethod=1
FlMethod=2
no_data=-999
buffer=1.5e5
var_name='tmin'
var_unit='degC'
var_long_name='minimum daily temperature'
corrNeg='.False.'
distZ='.False.'
;;
'rh')
'rh')
datatype="Relative Humidity"
FlVar=3
FlMethod=2
no_data=-9
buffer=1.5e5
;;
'windspeed')
'windspeed')
datatype="horizontal windspeed"
FlVar=4
FlMethod=1
no_data=-9
buffer=1.5e5
;;
*) 'ERROR: Please enter a valid prefix!'
*) 'ERROR: Please enter a valid var!'
esac
# *************************************************************
......@@ -124,92 +128,157 @@ esac
# 3 RU
# 4 windspeed
#
# FalgMthType - FlMehtod 1 - EDK
# 2 - OK
#
# Variogram Type - varType (vType)
# varType 1 : composed: nugget + spherical + sill
# FalgMthType - FlMehtod 2 - EDK
# 1 - OK
# 0 - no interpolation
# Variogram Type - var_type (vType)
# var_type 1 : composed: nugget + spherical + sill
# 2 : composed: nugget + exponential + sill
#
# VARIOGRAM-estimate: true: calculation of variogram parameters
# false: read parameters from file
#
# *************************************************************
cd ${rundir}
joblabel="edk_"${prefix}_${edkyStart}_${edkyEnd}
outfile=${prefix}_${edkyStart}_${edkyEnd}'.output'
# check if the blocks do exist
echo "Processing year : ${edkyStart}"
if [ -f "${outpath}${edkyStart}.bin" ] ; then
echo "***CYCLING: year.bin already existing"
continue
# exit 2
fi
outpath=${outpath}/
# check if the blocks do exist
# check if the blocks do exist
if [ -d ${outpath} ] ; then
echo "Path: " ${outpath}" does alreaddy exists"
continue # exit 2
echo "Path: " ${outpath}" does already exists"
else
mkdir -p ${outpath}
mkdir -p ${outpath}
fi
# create main file in the output folder
mainfile=${outpath}/'main.dat'
#mainfile='main.dat'
# create main file in the output folder
mainfile=${outpath}/'edk.nml'
#mainfile='main.dat'
# create mainfile containing filedir's etc.
if [ -f ${mainfile} ] ; then rm ${mainfile} ; fi
cat > ${mainfile} << EOF
!EDK-Grid***Main configuration file for ${datatype} for ${catchment}***
[[ -f "${outpath}/edk.nml" ]] && rm "${outpath}/edk.nml"
cat > "${outpath}/edk.nml" << EOF
!EDK-Grid***Main configuration file for Precipitation for GER***
&mainVars ! namelist mainpaths
flagMthTyp = ${FlMethod}
flagVarTyp = ${FlVar}
! -------------------------------------------------------------------
! ----------------- INPUT SPECIFICATIONS ----------------------------
!
! DataPathIn contains the path to the meteorological input files
! it can be a netcdf file, then a 3-dim field is expected and
! further coordinate and variable names are expected
DataPathIn = "${stationpath}"
!
! value in the meteorological input files that is used as missing value
noDataValue = ${no_data}
DataPathIn = "${inpath}"
fNameDEM = "${DEMfile}"
DataPathOut = "${outpath}"
fNameSTA = "${LuT}"
! Specifications of variable and coordinate names in case DataPathIn
! is a netcdf file
ncIn_variable_name = "${var_name}"
ncIn_dem_variable_name = "dem"
! name of coordinates, these must be in [m]
ncIn_yCoord_name = "northing"
ncIn_xCoord_name = "easting"
!
! Correct interpolated negative values to zero
! (set for precipitation to true !)
correctNeg = ${corrNeg}
!
! Set values further away than the distance threshold to zero
distZero = ${distZ}
!
cellFactor = ${cellfactor}
fNameDEM = "${dem}"
! name of coordinates (northing and easting) in [m]
ncOut_dem_yCoord_name = "northing"
ncOut_dem_xCoord_name = "easting"
! name of coordinates (latitude and longitude) in deg
ncOut_dem_Latitude = "latitude"
ncOut_dem_Longitude = "longitude"
! name of the DEM variable in the netcdf file
ncOut_dem_variable_name = "dem"
!
DataPathOut = "${outpath}"
FileOut = "${var_name}.nc"
!
! Name of Look Up Table of Station data
fNameSTA = "${lut}"
DataConvertFactor = ${dataconv}
outputformat = "${dataformat}"
! The value that should be added to the data in the netcdf file
OffSet = 0
![FINAL VALUE = NETCDF VALUE * DataConvertFactor + OffSet]
!
! -------------------------------------------------------------------
! ------------ PROCESSING PERIOD ------------------------------------
yStart = ${start_year}
mStart = ${start_month}
dStart = ${start_day}
yEnd = ${end_year}
mEnd = ${end_month}
dEnd = ${end_day}
! Number of Time Buffering Days (Divides the EDK processing into chunks accross the time dimension)
tBuffer = 30
!
! -------------------------------------------------------------------
! ------------ INTERPOLATION METHOD ---------------------------------
!
! InterMth = 2 -> EDK
! InterMth = 1 -> OK
! InterMth = 0 -> No interpolation
InterMth = ${FlMethod}
! maximum search distance for interpolation [m]
maxDist = ${buffer}
!
! -------------------------------------------------------------------
! ----------------- VARIOGRAM ESTIMATION ----------------------------
flagVario = ${vario}
!
! number of variogram parameters
nParam = ${var_param}
! type of theoretical variogramm (1=spherical, 2 = exponential)
vType = ${var_type}
!
! file name where to store the variogram parameters
! (if flagVario is false, variogram parameters )
! (for interpolation are read in here: )
fNameVario = "${vario_fname}"
dh = ${binsize} ! binsize for variogram [m]
hMax = ${hmax} ! max distance h for variogram estimation [m]
! -------------------------------------------------------------------
! --------------- NC OUTPUT SPECIFICATION ----------------------------
author_name = 'Friedrich Boeing'
projection_name = 'EPSG: 4326'
invert_y = .True. ! (set True if working with mHM input data!)
variable_name = "${var_name}"
variable_unit = "${var_unit}"
variable_long_name = "${var_long_name}"
flagEDK = ${edkesti}
yStart = ${edkyStart}
mStart = ${edkmStart}
dStart = ${edkdStart}
yEnd = ${edkyEnd}
mEnd = ${edkmEnd}
dEnd = ${edkdEnd}
maxDist = ${buffer}
flagVario = ${vario}
vType = ${varType}
nParam = ${varParam}
fNameVario = "${varioFname}" ! path+filename
dh = ${binsize} ! binsize for variogram
hMax = ${hMAx} ! max distance h
variable_standard_name = 'precipitation_flux'
variable_calendar_type = 'proleptic_gregorian'
! -------------------------------------------------------------------
! -------------------------------------------------------------------
/ !END*******Main Config***********
EOF
joblabel="edk_"${var}_${start_date}_${end_date}
outfile=${var}_${start_date}_${end_date}'.output'
# create Readme file in output directory
printf "Interpolation Started " > ${outpath}/README
date "+on %m/%d/%Y at %H:%M:%S %n" >> ${outpath}/README
cat ${mainfile} >> ${outpath}/README
printf "\n \n>****Variogram Parameters \n" >> ${outpath}/README
cat ${varioFname} >> ${outpath}/README
cat ${vario_fname} >> ${outpath}/README
if [ ${interactive} == "no" ] ; then
# create script for submission to the cluster
fSubmit="${outpath}/qsub_tmp.sh"
if [ -f ${fSubmit} ] ; then rm ${fSubmit} ; fi
cat > ${fSubmit} << EOF
#!bin/bash
# create script for submission to the cluster
fSubmit="${outpath}/qsub_tmp.sh"
if [ -f ${fSubmit} ] ; then rm ${fSubmit} ; fi
cat > ${fSubmit} << EOF
#!/bin/bash
#-------------------------------
#\$ -S /bin/bash
#\$ -N ${joblabel}
......@@ -217,34 +286,49 @@ if [ ${interactive} == "no" ] ; then
#\$ -j y
#\$ -l h_rt=${rtime}
#\$ -l h_vmem=${vmem}
#\$ -l highmem=${hmem}
#\$ -cwd
#\$ -binding linear:1
#\$ -pe smp ${OMP_NTHREAD}
#\$ -m ea # mail notification in case of job ended abortion
#\$ -M ${mail} # mail address
#-------------------------------
ulimit -s unlimited
export OMP_NUM_THREADS=1
cd ${outpath}
export OMP_NUM_THREADS=${OMP_NTHREAD}
cd ${outpath} || exit
ml foss/2018b
ml netCDF-Fortran
time ./edk
rm edk
rm main.dat
# rm edk
# rm edk.nml
EOF
#rm ./edk
# copy executeable to the output directory to run it from there (uniqe executeable)
cp edk ${outpath}
cd ${outpath}
qsub qsub_tmp.sh
ln -fs ${EDK_PROG_PATH} ${outpath}
cd ${outpath}
if [[ $qsub_manual == "true" ]]; then
echo "..!manual job submission to cluster!..."
else
qsub qsub_tmp.sh
echo "Finished submission"
fi
else
# run program on frontend, lauch it from the output directory
cp edk ${outpath}
cd ${outpath}
time ./edk
rm edk
rm main.dat
# run program on frontend, lauch it from the output directory
ln -fs ${EDK_PROG_PATH} ${outpath}
cd ${outpath}
export OMP_NUM_THREADS=${OMP_NTHREAD}
ml foss/2018b
ml netCDF-Fortran
time ./edk
# rm edk
# rm edk.nml
echo "Finished submission"
fi
# change back to root directory before next job starts/is submitted
cd ${rundir}
cd ${PWD}
echo "Finished submission"
exit 0
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment