The UFZ services GitLab and Mattermost will be unavailable on Monday, December 06 from 06:00 AM to 08:00 AM due to maintenance work.

Commit fa1b5f33 authored by David Schäfer's avatar David Schäfer
Browse files

work around the 1000 elements per query restriction

parent 52d89933
......@@ -4,7 +4,6 @@ import (
s "dmpapi/settings"
"fmt"
"math"
"strconv"
"strings"
"time"
)
......@@ -62,39 +61,34 @@ func (db *DataBase) GetSensorDataLevel1(projectId, sensorId, start, limit int, s
}
func (db *DataBase) GetLoggerDataLevel1Cached(projectId, loggerId, start, limit int, startDate, endDate time.Time) (*L1Data, error) {
// join the sensor information manually, to safe database cycles
sensors, err := db.GetSensorsByLoggerID(projectId, loggerId, 0, math.MaxInt64)
sensorIds, lut, err := db.querySensorInformation(projectId, loggerId)
if err != nil {
return nil, err
}
if len(*sensors) == 0 {
return nil, fmt.Errorf("no such logger found: %v", loggerId)
}
sensorIds := []string{}
lut := map[int]SensorModel{}
for _, row := range *sensors {
sensorIds = append(sensorIds, strconv.Itoa(row.ID))
lut[row.ID] = row
}
out := L1Data{}
for _, chunk := range sensorIds {
data, err := db.getL1Data(
fmt.Sprintf(L1_CACHE_QUERY, strings.Join(chunk, ",")),
startDate.Format(s.DATE_FORMAT), endDate.Format(s.DATE_FORMAT),
start, limit,
)
data, err := db.getL1Data(
fmt.Sprintf(L1_CACHE_QUERY, strings.Join(sensorIds, ",")),
startDate.Format(s.DATE_FORMAT), endDate.Format(s.DATE_FORMAT),
start, limit,
)
if err != nil {
return nil, err
}
if err != nil {
return data, err
for _, row := range *data {
sensor := lut[row.SensorID]
row.SensorName = sensor.Name
row.SensorLabel = sensor.Label
out = append(out, row)
}
}
for i, row := range *data {
sensor := lut[row.SensorID]
row.SensorName = sensor.Name
row.SensorLabel = sensor.Label
(*data)[i] = row
}
return data, nil
return &out, nil
}
func (db *DataBase) GetLoggerDataLevel1(projectId, loggerId, start, limit int, startDate, endDate time.Time) (*L1Data, error) {
......
......@@ -4,7 +4,6 @@ import (
s "dmpapi/settings"
"fmt"
"math"
"strconv"
"strings"
"time"
)
......@@ -36,41 +35,33 @@ func (d *L2Data) Length() int {
}
func (db *DataBase) GetLoggerDataLevel2Cached(projectId, loggerId, start, limit int, startDate, endDate time.Time) (*L2Data, error) {
sensors, err := db.GetSensorsByLoggerID(projectId, loggerId, 0, math.MaxInt64)
sensorIds, lut, err := db.querySensorInformation(projectId, loggerId)
if err != nil {
return nil, err
}
if len(*sensors) == 0 {
return nil, fmt.Errorf("no such logger found: %v", loggerId)
}
out := L2Data{}
for _, chunk := range sensorIds {
data, err := db.getL2Data(
fmt.Sprintf(L2_CACHE_QUERY, strings.Join(chunk, ",")),
startDate.Format(s.DATE_FORMAT), endDate.Format(s.DATE_FORMAT),
start, limit,
)
sensorIds := []string{}
lut := map[int]SensorModel{}
for _, sensor := range *sensors {
sensorIds = append(sensorIds, strconv.Itoa(sensor.ID))
lut[sensor.ID] = sensor
}
if err != nil {
return nil, err
}
data, err := db.getL2Data(
fmt.Sprintf(L2_CACHE_QUERY, strings.Join(sensorIds, ",")),
startDate.Format(s.DATE_FORMAT),
endDate.Format(s.DATE_FORMAT),
start,
limit,
)
for _, row := range *data {
sensor := lut[row.SensorID]
row.SensorName = sensor.Name
row.SensorLabel = sensor.Label
out = append(out, row)
}
if err != nil {
return data, err
}
for i, row := range *data {
sensor := lut[row.SensorID]
row.SensorName = sensor.Name
row.SensorLabel = sensor.Label
(*data)[i] = row
}
return data, nil
return &out, nil
}
func (db *DataBase) GetLoggerDataLevel2(projectId, loggerId, start, limit int, startDate, endDate time.Time) (*L2Data, error) {
......
package db
import (
"fmt"
"math"
"strconv"
)
type locationModel struct {
Lon NullFloat64 `db:"LON" json:"lon"`
Lat NullFloat64 `db:"LAT" json:"lat"`
......@@ -41,6 +47,36 @@ func (d *SensorData) Length() int {
return len(*d)
}
func (db *DataBase) querySensorInformation(projectId, loggerId int) ([][]string, map[int]SensorModel, error) {
// return sensor_ids and a id based lookup table into the query results
sensors, err := db.GetSensorsByLoggerID(projectId, loggerId, 0, math.MaxInt64)
if err != nil {
return nil, nil, err
}
if len(*sensors) == 0 {
return nil, nil, fmt.Errorf("no such logger found: %v", loggerId)
}
// we can only query 1000 sensor_ids at once, if we use the
// query formatting approach instead of (multiple) subqueries
sensorIds := [][]string{}
lut := map[int]SensorModel{}
j := -1
for i, row := range *sensors {
if i%1000 == 0 {
j += 1
sensorIds = append(sensorIds, []string{})
}
sensorIds[j] = append(sensorIds[j], strconv.Itoa(row.ID))
lut[row.ID] = row
}
return sensorIds, lut, nil
}
func (db *DataBase) GetSensorsAll(projectId, start, limit int) (*SensorData, error) {
return db.querySensors(
SENSORS_QUERY,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment