Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions services/water_level_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
SAMPLE_QC_TYPE,
depth_to_water_error,
measuring_point_height_conflict_message,
observation_release_status,
reconcile_measuring_point_height,
)
from pydantic import ValidationError
Expand Down Expand Up @@ -385,7 +386,15 @@ def _create_records(
notes=None,
)
sample = Sample(field_activity=field_activity)
observation = Observation(sample=sample)
observation = Observation(
sample=sample,
# Copy the well's status, because the public water level
# views check the observation's status too. It is set only
# where the observation is new (here and in the branch
# below), so a re-import never overwrites a status someone
# set by hand.
release_status=observation_release_status(row.well.release_status),
)
session.add(field_event)
session.add(field_activity)
session.add(sample)
Expand All @@ -395,7 +404,14 @@ def _create_records(
field_event = field_activity.field_event
observation = _find_existing_observation(sample, parameter_id)
if observation is None:
observation = Observation(sample=sample)
# A new observation on an existing sample: copy the well's
# status for the same reason as above.
observation = Observation(
sample=sample,
release_status=observation_release_status(
row.well.release_status
),
)
session.add(observation)

field_event.event_date = row.field_event_dt
Expand Down
11 changes: 10 additions & 1 deletion tests/test_data_migrations.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@
from db.stratigraphic_interval import StratigraphicInterval
from db.engine import session_ctx
from db.transducer import TransducerObservation
import services.water_level_csv as water_level_csv
from services.water_level_csv import bulk_upload_water_levels
from tests import get_parameter_id

Expand Down Expand Up @@ -1963,9 +1964,17 @@ def test_water_level_release_status_copies_csv_well_status(


def test_water_level_release_status_publishes_a_real_csv_import(
water_level_release_status, water_well_thing
water_level_release_status, water_well_thing, monkeypatch
):
"""End to end through the importer, so the footprint is the real one."""
# The importer now copies the well's status itself (BDMS-1560). The
# backfill is for observations imported before that, which were left
# draft, so make the importer behave as it did then. Patching the status
# keeps the draft in observation_version too, which the backfill's history
# check reads.
monkeypatch.setattr(
water_level_csv, "observation_release_status", lambda _status: "draft"
)
header = (
"field_staff,well_name_point_id,field_event_date_time,"
"measurement_date_time,sampler,sample_method,mp_height,level_status,"
Expand Down
164 changes: 163 additions & 1 deletion tests/test_water_level_csv_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from decimal import Decimal
from types import SimpleNamespace

import pytest
from db import (
Contact,
FieldActivity,
Expand All @@ -13,6 +14,7 @@
)
from db.measuring_point_history import MeasuringPointHistory
from db.engine import session_ctx
from domain.samples import water_level_sample_name
from tests import get_parameter_id
from services.water_level_csv import (
_build_sample_name,
Expand All @@ -21,7 +23,7 @@
_validate_depth_to_water_against_well,
bulk_upload_water_levels,
)
from sqlalchemy import select
from sqlalchemy import select, text


def _build_well(
Expand Down Expand Up @@ -853,3 +855,163 @@ def test_bulk_upload_water_levels_reports_duplicate_well_name_matches():
if well is not None:
session.delete(well)
session.commit()


def _single_row_csv(well_name: str) -> bytes:
"""One valid groundwater level row, measured at 2025-02-15 17:30 UTC."""
header = [
"field_staff",
"well_name_point_id",
"field_event_date_time",
"measurement_date_time",
"sampler",
"sample_method",
"mp_height",
"level_status",
"depth_to_water_ft",
"data_quality",
"water_level_notes",
]
values = [
"A Lopez",
well_name,
"2025-02-15T08:00:00-07:00",
"2025-02-15T10:30:00-07:00",
"A Lopez",
"electric tape",
"1.5",
"Water level not affected",
"7.0",
"Water level accurate to within two hundreths of a foot",
"Initial measurement",
]
return "\n".join([",".join(header), ",".join(values)]).encode("utf-8")


def _set_well_release_status(well_id: int, release_status: str) -> None:
with session_ctx() as session:
session.get(Thing, well_id).release_status = release_status
session.commit()


def _count_in_ogc_waterlevels(session, observation_id: int) -> int:
# The view prefixes manual observation ids with "m-" to keep them apart
# from transducer observation ids.
return session.execute(
text("SELECT count(*) FROM ogc_waterlevels WHERE id = :id"),
{"id": f"m-{observation_id}"},
).scalar_one()


@pytest.mark.parametrize(
"well_status, expected",
[
("public", "public"),
("private", "private"),
("draft", "draft"),
# Not a publication decision, so the observation stays draft.
("provisional", "draft"),
],
)
def test_bulk_upload_water_levels_sets_new_observation_release_status_from_well(
water_well_thing, well_status, expected
):
"""The public water level views need the observation public as well as the well."""
_set_well_release_status(water_well_thing.id, well_status)

result = bulk_upload_water_levels(_single_row_csv(water_well_thing.name))

assert result.exit_code == 0, result.payload
observation_id = result.payload["water_levels"][0]["observation_id"]
with session_ctx() as session:
observation = session.get(Observation, observation_id)
sample = observation.sample

assert observation.release_status == expected
# Only the observation follows the well; the visit records stay draft.
assert sample.release_status == "draft"
assert sample.field_activity.release_status == "draft"
assert sample.field_activity.field_event.release_status == "draft"

# The fix exists so the public EDR collection includes these
# observations, so check the collection itself.
published = _count_in_ogc_waterlevels(session, observation_id)
assert published == (1 if expected == "public" else 0)


def test_bulk_upload_water_levels_reimport_keeps_existing_observation_release_status(
water_well_thing,
):
"""A re-import must not undo a status someone set on the observation by hand."""
csv_content = _single_row_csv(water_well_thing.name)
first = bulk_upload_water_levels(csv_content)
assert first.exit_code == 0, first.payload
observation_id = first.payload["water_levels"][0]["observation_id"]

# The well is public, so a re-import that copied the well's status again
# would turn this back to public.
with session_ctx() as session:
session.get(Observation, observation_id).release_status = "private"
session.commit()

second = bulk_upload_water_levels(csv_content)

assert second.exit_code == 0, second.payload
assert second.payload["water_levels"][0]["observation_id"] == observation_id
with session_ctx() as session:
assert session.get(Observation, observation_id).release_status == "private"


@pytest.mark.parametrize("well_status", ["public", "private"])
def test_bulk_upload_water_levels_sets_release_status_on_new_observation_for_existing_sample(
water_well_thing, well_status
):
"""An existing sample without a groundwater level observation gets a new one,
and that new observation copies the well's status too."""
_set_well_release_status(water_well_thing.id, well_status)
ph_parameter_id = get_parameter_id("pH", "Field Parameter")
measured_at = datetime(2025, 2, 15, 17, 30, tzinfo=timezone.utc)

with session_ctx() as session:
# Load the well rather than merging the fixture object, whose stale
# release_status would overwrite the one set above.
well = session.get(Thing, water_well_thing.id)
field_event = FieldEvent(
thing=well,
event_date=datetime(2025, 2, 15, 15, 0, tzinfo=timezone.utc),
)
field_activity = FieldActivity(
field_event=field_event, activity_type="groundwater level"
)
# Same name the importer builds for the CSV row, so it reuses this sample.
sample = Sample(
field_activity=field_activity,
sample_date=measured_at,
sample_name=water_level_sample_name(water_well_thing.name, measured_at),
sample_matrix="groundwater",
sample_method="Electric tape measurement (E-probe)",
qc_type="Normal",
)
ph_observation = Observation(
sample=sample,
observation_datetime=measured_at,
parameter_id=ph_parameter_id,
value=7.2,
unit="dimensionless",
)
session.add_all([field_event, field_activity, sample, ph_observation])
session.commit()
sample_id = sample.id
ph_observation_id = ph_observation.id

result = bulk_upload_water_levels(_single_row_csv(water_well_thing.name))

assert result.exit_code == 0, result.payload
imported = result.payload["water_levels"][0]
assert imported["sample_id"] == sample_id
with session_ctx() as session:
observation = session.get(Observation, imported["observation_id"])
assert observation.id != ph_observation_id
assert observation.release_status == well_status
# The observation that was already on the sample is not an import.
assert session.get(Observation, ph_observation_id).release_status == "draft"
Loading