Skip to content

electricblue

ElectricBlue

https://electricblue.eu/envloggers

ElectricBlue is a non-profit technology transfer startup creating research-oriented solutions for the scientific community

csv(path, encoding='utf-8')

ElectricBlue csv data format parser

Parameters:

Name Type Description Default
path str

path to the csv file to parse

 required
encoding str='UTF-8'

file encoding

 'utf-8'

Returns:

Name Type Description
dataset xarray.Dataset

xarray dataset
Source code in ocean_data_parser/parsers/electricblue.py 
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
def csv(
    path: str,
    encoding: str = "utf-8",
) -> xarray.Dataset:
    """ElectricBlue csv data format parser

    Args:
        path (str): path to the csv file to parse
        encoding (str='UTF-8', optional): file encoding
    Returns:
        dataset: xarray dataset
    """
    with open(path, encoding=encoding) as f:
        line = True
        metadata = default_global_attribute
        metadata["source_file"] = path

        while line:
            line = f.readline()
            metadata["source_file_header"] += line
            line = line.strip()

            if re.match(r"^[-,\s]+$", line):
                continue
            elif line.startswith("time,"):
                break
            else:
                items = line.split(", ", 1)
                key = items[0]
                value = items[1] if len(items) == 2 else ""

                attr = re.sub(r"[\s\[\]\(\)]+", "_", key.lower())

                # cast value
                if re.match(r"^[+-]*\d+$", value):
                    value = int(value)
                elif re.match(r"^[+-]*\d+\.\d+$", value):
                    value = float(value)

                metadata[attr] = value

        columns = line.split(",")
        time_zone = metadata.pop("time_zone")
        df = pd.read_csv(
            f,
            sep=",",
            header=None,
            names=columns,
            converters={0: lambda x: pd.to_datetime(x + time_zone, utc=True)},
        )
        if len(df) != metadata["samples"]:
            logger.warning(
                "Parsed data samples=%s do not match expected samples=%s",
                str(len(df)),
                metadata["samples"],
            )

        # Convert to xarray dataset
        ds = df.to_xarray()

        # Global attributes
        ds.attrs = metadata
        ds.attrs.update(
            {
                "instrument_type": ds.attrs.get("envlogger_version"),
                "instrument_sn": ds.attrs.get("serial_number"),
            }
        )
        ds["latitude"] = ds.attrs["lat"]
        ds["longitude"] = ds.attrs["long"]

        # Variables attributes
        for var in ds:
            if var in default_variable_attributes:
                ds[var].attrs = default_variable_attributes[var]
        ds["temp"].attrs["units"] = ds.attrs.pop("temperature")
        ds = standardize_dataset(ds)
        return ds