Hi I am using the CDS toolbox to download and average ERA-5 variables. I am running into intermittent problems with errors during retrievals. Samples of those failures are in my "FAILED REQUESTS" log: A
A sample is provided here:
Request ID: 0ee966ba-5a47-42a9-acde-0265378f2185
It is a little hard to read but there appears to be some conversion error?
The same workflow executes fine for other months. I am submitting this workflow via the CDSAPI from a remote computer.
Any suggestions o how to fix this?
Thanks
Axel
tool.toolbox.orchestrator.run_workflow
Request body:
{ "args": [ { "code": "#!/usr/bin/env python3\n# -*- coding: utf-8 -*-\n\"\"\"\nCreated on Fri Apr 17 12:54:44 2020\n\n@author: axel\n\"\"\"\n# deals with accumulated values \nimport cdstoolbox as ct\n\n@ct.application(title='Daily Download')\n@ct.output.download()\ndef application(varname,year, month):\n tmonth=int(month)\n tyear=year\n selector=str(tyear) + \"-\" + str(tmonth).zfill(2)\n print(\"Axel: Selector \", selector)\n if tmonth == 12: \n syears=[str(tyear),str(tyear+1)]\n smonths=[str(tmonth).zfill(2),str(1).zfill(2)]\n else:\n syears=str(tyear)\n smonths=[str(tmonth).zfill(2),str(tmonth+1).zfill(2)]\n print(\"Axel: \",smonths,syears)\n # month can only be a single number in this case.. could sort this out to for multiple\n data = ct.catalogue.retrieve(\n 'reanalysis-era5-single-levels',\n {\n 'variable': varname,\n 'product_type': 'reanalysis',\n 'year': syears,\n 'month': smonths, \n 'day': ['01','02','03', \n '04', '05', '06',\n '07', '08', '09',\n '10', '11', '12',\n '13', '14', '15',\n '16', '17', '18',\n '19', '20', '21',\n '22', '23', '24',\n '25', '26', '27',\n '28', '29', '30',\n '31'],\n 'time': [\n '00:00', '01:00', '02:00',\n '03:00', '04:00', '05:00',\n '06:00', '07:00', '08:00',\n '09:00', '10:00', '11:00',\n '12:00', '13:00', '14:00',\n '15:00', '16:00', '17:00',\n '18:00', '19:00', '20:00',\n '21:00', '22:00', '23:00',\n ],\n }\n )\n data=ct.operator.div(data,3600.)\n# data=ct.climate.daily_mean(data)\n daily_mean=ct.cube.resample(data,freq='D',how='mean',closed='right',dim='time')\n sel=ct.cube.select(daily_mean,time=selector)\n# print(daily_mean)\n return sel", "args": [], "kwargs": { "varname": "surface_latent_heat_flux", "year": 2012, "month": "12" }, "workflow_name": "application" } ], "kwargs": {} }
Reason: Traceback (most recent call last): File "/opt/cdstoolbox/cdscompute/cdscompute/cdshandlers/services/handler.py", line 49, in handle_request result = cached(context.method, proc, context, context.args, context.kwargs) File "/opt/cdstoolbox/cdscompute/cdscompute/caching.py", line 108, in cached result = proc(context, *context.args, **context.kwargs) File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 118, in __call__ return p(*args, **kwargs) File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 59, in __call__ return self.proc(context, *args, **kwargs) File "/home/cds/cdsservices/services/retrieve.py", line 191, in execute time_dimensions=time_dimensions, read_attributes=read_attributes File "/opt/cdstoolbox/cdscompute/cdscompute/context.py", line 270, in call return c.call(service, *args, **kwargs).value File "/opt/cdstoolbox/cdsworkflows/cdsworkflows/future.py", line 76, in value raise self._result cdsworkflows.error.ClientError: {'traceback': 'Traceback (most recent call last):\n File "/opt/cdstoolbox/cdscdm/cdscdm/gridded.py", line 194, in convert_variable\n var = units.convert_units(var, default_att[\'units\'])\n File "/opt/cdstoolbox/cdscdm/cdscdm/units.py", line 41, in convert_units\n target = source.astype(cast).load()\n File "/usr/local/lib/python3.6/site-packages/xarray/core/common.py", line 1371, in astype\n dask="allowed",\n File "/usr/local/lib/python3.6/site-packages/xarray/core/computation.py", line 1110, in apply_ufunc\n keep_attrs=keep_attrs,\n File "/usr/local/lib/python3.6/site-packages/xarray/core/computation.py", line 262, in apply_dataarray_vfunc\n result_var = func(*data_vars)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/computation.py", line 610, in apply_variable_ufunc\n for arg, core_dims in zip(args, signature.input_core_dims)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/computation.py", line 610, in <listcomp>\n for arg, core_dims in zip(args, signature.input_core_dims)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/computation.py", line 522, in broadcast_compat_data\n data = variable.data\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 359, in data\n return self.values\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 510, in values\n return _as_array_or_item(self._data)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/variable.py", line 272, in _as_array_or_item\n data = np.asarray(data)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 560, in __array__\n return np.asarray(array[self.key], dtype=None)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 685, in __array__\n self._ensure_cached()\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 682, in _ensure_cached\n self.array = NumpyIndexingAdapter(np.asarray(self.array))\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 655, in __array__\n return np.asarray(self.array, dtype=dtype)\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/_asarray.py", line 85, in asarray\n return array(a, dtype, copy=False, order=order)\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 560, in __array__\n return np.asarray(array[self.key], dtype=None)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/cfgrib_.py", line 24, in __getitem__\n key, self.shape, indexing.IndexingSupport.OUTER, self._getitem\n File "/usr/local/lib/python3.6/site-packages/xarray/core/indexing.py", line 845, in explicit_indexing_adapter\n result = raw_indexing_method(raw_key.tuple)\n File "/usr/local/lib/python3.6/site-packages/xarray/backends/cfgrib_.py", line 29, in _getitem\n return self.array[key]\n File "/usr/local/lib/python3.6/site-packages/cfgrib/dataset.py", line 329, in __getitem__\n array_field = np.full(array_field_shape, fill_value=np.nan, dtype=\'float32\')\n File "/usr/local/lib64/python3.6/site-packages/numpy/core/numeric.py", line 332, in full\n a = empty(shape, dtype, order)\nMemoryError: Unable to allocate array with shape (2976, 721, 1440) and data type float32\n\nThe above exception was the direct cause of the following exception:\n\nTraceback (most recent call last):\n File "/opt/cds/cdsinf/python/lib/cdsinf/runner/dispatcher.py", line 474, in handle_request\n context.get("method_config", None))\n File "/opt/cdstoolbox/cdscompute/cdscompute/cdshandlers/services/handler.py", line 49, in handle_request\n result = cached(context.method, proc, context, context.args, context.kwargs)\n File "/opt/cdstoolbox/cdscompute/cdscompute/caching.py", line 108, in cached\n result = proc(context, *context.args, **context.kwargs)\n File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 118, in __call__\n return p(*args, **kwargs)\n File "/opt/cdstoolbox/cdscompute/cdscompute/services.py", line 59, in __call__\n return self.proc(context, *args, **kwargs)\n File "/home/cds/cdsservices/services/cdm_translate.py", line 61, in execute\n retrieve_kwargs={\'time_dimensions\': time_dimensions, \'read_attributes\': read_attributes}\n File "/opt/cdstoolbox/cdscdm/cdscdm/gridded.py", line 701, in open_dataset\n datasets = transform_dataset(dataset, dataset_spec, strict=strict, remove_grib=remove_grib, keep_attrs=keep_attrs)\n File "/opt/cdstoolbox/cdscdm/cdscdm/gridded.py", line 486, in transform_dataset\n var = transform_variable(var, var_spec)\n File "/opt/cdstoolbox/cdscdm/cdscdm/gridded.py", line 228, in transform_variable\n raise ValueError(\'Conversion error for variable %r (%r)\' % (var.name, original_var_name)) from exc\nValueError: Conversion error for variable \'hfls_accumulated\' (\'slhf\')\n'}