"""raw data + config + measured overrides -> build/corridor.json (the canonical corridor dataset). Every number carries: source (measured | derived | assumed), evidence (why), ref (where from). Measured files in data/measured/ always win over derived/assumed values. Measured override files (all optional, CSV with header): data/measured/approach_volumes.csv : light_id,approach,q_vph,L,T,R,note (L/T/R shares optional) """ import csv, json, math, statistics, tomllib, datetime, pathlib, sys ROOT = pathlib.Path(__file__).resolve().parents[1] RAW, MEAS, BUILD, CFG = ROOT/"data/raw", ROOT/"data/measured", ROOT/"build", ROOT/"config" DIRS = ["NB", "SB", "EB", "WB"] SHIFT = {"NB": (-0.0012, 0), "SB": (0.0012, 0), "EB": (0, -0.0015), "WB": (0, 0.0015)} # point on the approach leg def load_toml(name): with open(CFG/name, "rb") as f: return tomllib.load(f) def haversine(a, b): R = 6371000.0 la1, lo1, la2, lo2 = map(math.radians, (a[0], a[1], b[0], b[1])) return 2*R*math.asin(math.sqrt(math.sin((la2-la1)/2)**2 + math.cos(la1)*math.cos(la2)*math.sin((lo2-lo1)/2)**2)) def county_stations(): feats = json.load(open(RAW/"county/county-count-locations-raw.json"))["features"] hist = json.load(open(RAW/"county/county-count-history-raw.json"))["features"] st = {} for f in feats: a = f["attributes"] st[int(a["Location_ID"])] = dict(id=int(a["Location_ID"]), on=a["On_Street"], at=a["At_Street"], side=a["Direction_From"], year=a["Most_Recent_Year"], awt=a["Most_Recent_AWT"], pk=a["PkHrVol"], pk_time=f'{a["PkHrTime"]} {a["PkHrAMPM"]}', lat=f["geometry"]["y"], lon=f["geometry"]["x"]) return st def modot_rows(): return list(csv.DictReader(open(RAW/"modot/modot_aadt_2025_corridor.csv"))) def modot_dir_ratio(rows, road, pt, d): """Share of daily traffic going in direction d (NB/SB/EB/WB) on `road` near point pt, from MoDOT directional AADT.""" inv = {"NB": "E", "SB": "W", "EB": "E", "WB": "W"}[d] other = {"E": "W", "W": "E"}[inv] def nearest(dirn): cand = [r for r in rows if road in r["road"] and r["inv_dir"] == dirn] def dist(r): mid = ((float(r["start_lat"])+float(r["end_lat"]))/2, (float(r["start_lon"])+float(r["end_lon"]))/2) return haversine(pt, mid) return min(cand, key=dist) a, b = nearest(inv), nearest(other) return float(a["dir_aadt"])/(float(a["dir_aadt"])+float(b["dir_aadt"])), a, b def county_counts(lid, hour): """Measured County turning-movement count for one light, summed over [hour, hour+1h). None if no file.""" files = sorted((MEAS/"county-counts").glob(f"{lid}-*.csv")) files = [f for f in files if not f.name.endswith(".totals.csv")] if not files: return None f = files[-1]; h0 = int(hour[:2])*60+int(hour[3:]) agg = {} for r in csv.DictReader(l for l in open(f) if not l.startswith("#")): hh, mm = map(int, r["hour"].split(":")); t = hh*60+mm if h0 <= t < h0+60: a = agg.setdefault(r["approach"], dict(L=0, T=0, R=0)) for k in ("L", "T", "R"): a[k] += int(r[k]) return dict(file=f.name, date=f.stem.split("-", 3)[-1] if f.stem.count("-") >= 3 else "", approaches=agg) def county_timing(lid, pattern, basis="signal_sequence"): """Measured County timing for one light, reduced to the model's two-phase form. None if no sheet.""" jf = MEAS/"county_timing.json" if not jf.exists(): return None T = json.load(open(jf)).get(lid) if not T or pattern not in T["patterns"]: return None P = T["patterns"][pattern]; ph = {int(k): v for k, v in T["phases"].items()}; sp = {int(k): v for k, v in P["splits"].items()} main = [n for n in sp if "Big Bend" in ph.get(n, {}).get("movement", "")] side = [n for n in sp if n not in main] ring = lambda ns, r: [n for n in ns if (n <= 4) == (r == 1)] block = lambda ns: max(sum(sp[n] for n in ring(ns, 1)), sum(sp[n] for n in ring(ns, 2))) thru = lambda ns: [n for n in ns if "LT" not in ph[n]["movement"]] or ns m_t, s_t = thru(main), thru(side) ya = lambda n: ph[n]["yellow"] + (ph[n]["all_red"] or 0) nm, ns = max(m_t, key=ya), max(s_t, key=ya) if basis == "signal_sequence": # every phase in the block clears; critical ring = the larger total Lm = max(sum(ya(n) for n in ring(main, r)) for r in (1, 2)) Ls = max(sum(ya(n) for n in ring(side, r)) for r in (1, 2)) else: Lm, Ls = ya(nm), ya(ns) lead_left = sum(sp[n] for n in main if "LT" in ph[n]["movement"]) / max(1, len({n <= 4 for n in main if "LT" in ph[n]["movement"]})) return dict(source="measured", sheet=T["file"], sheet_date=T["date"], pattern=pattern, C_s=P["cycle"], offset_county_s=P["offset"], splits=sp, main_phases=main, side_phases=side, main_block_s=block(main), side_block_s=block(side), clearance_main_s=Lm, clearance_side_s=Ls, L_s=round(Lm+Ls, 2), yellow_main_s=ph[nm]["yellow"], all_red_main_s=ph[nm]["all_red"] or 0.0, yellow_side_s=ph[ns]["yellow"], all_red_side_s=ph[ns]["all_red"] or 0.0, g_main_s=round(block(main)-Lm, 2), lead_left_s=round(lead_left, 2), main_start_s=round((P["offset"]-lead_left) % P["cycle"], 2), lost_time_basis=basis, note=f"2-phase reduction ({basis}): main green = Big Bend block minus its clearances; " "main start = County offset (start of Big Bend through green) minus the leading protected-left split") def main(): cor, asm = load_toml("corridor.toml"), load_toml("assumptions.toml") st, mrows = county_stations(), modot_rows() sig = {r["signal_id"]: r for r in csv.DictReader(open(RAW/"county/county-signals.csv"))} # corridor's own peak-hour factor ks = [st[i]["pk"]/st[i]["awt"] for i in cor["peak_factor"]["stations"] if st.get(i) and st[i]["awt"]] K = statistics.median(ks) measured = {} mfile = MEAS/"approach_volumes.csv" if mfile.exists(): for r in csv.DictReader(open(mfile)): measured[(r["light_id"], r["approach"])] = r sa, ts = asm["signal"], asm["turn_shares"] hour = cor.get("counts", {}).get("design_hour", "17:00"); pattern = cor.get("timing", {}).get("pattern", "37") counts_by_light = {lid: county_counts(lid, hour) for lid in cor["corridor"]["order"]} lights = [] for lid in cor["corridor"]["order"]: c = cor["lights"][lid] s = sig[lid] pos = (float(s["lat"]), float(s["lon"])) apps = {} for d in DIRS: pt = (pos[0]+SHIFT[d][0], pos[1]+SHIFT[d][1]) spec = c[d] main = d in ("NB", "SB") road = "BIG BEND" if main else c["cross_street"].upper().replace("BOULEVARD", "BLVD").replace("PARKWAY", "PKWY").replace("ROAD", "RD") ratio, ra, rb = modot_dir_ratio(mrows, road, pt, d) ratio = min(max(ratio + asm["direction_split"]["peak_skew"], 0.0), 1.0) if spec.startswith("county:"): sid = int(spec.split(":")[1]); x = st[sid] q = x["pk"]*ratio q_src, q_ev = "derived", (f'County station {sid} ({x["on"]} @ {x["at"]} {x["side"]}, {x["year"]}): peak hour {x["pk"]} veh/h two-way ' f'at {x["pk_time"]} x MoDOT direction share {ratio:.3f}') q_ref = f"county:{sid}" else: q = float(ra["dir_aadt"])*K q_src, q_ev = "derived", (f'MoDOT 2025 {ra["road"]} dir AADT {ra["dir_aadt"]} ({ra["seg_desc"]}) x corridor peak-hour factor {K:.3f}') q_ref = f'modot:{ra["road"]}:{ra["seg_desc"]}:{ra["inv_dir"]}' # lanes per direction from the MoDOT segment for this approach (measured); config default if blank lanes = int(float(ra["lanes"])) if ra.get("lanes") else (sa["lanes_main"] if main else sa["lanes_side_default"]) lanes_src = "measured (MoDOT segment lanes)" if d in c.get("lanes", {}): lanes, lanes_src = int(c["lanes"][d]), "measured (OpenStreetMap approach lanes, config override)" shares = dict(ts["main"] if main else ts["side_" + c.get("side_class", "minor")]); sh_src, sh_ev = "assumed", asm["turn_shares"]["evidence"] cc = counts_by_light.get(lid) if cc and d in cc["approaches"]: mv = cc["approaches"][d]; tot = mv["L"]+mv["T"]+mv["R"] q, q_src, q_ref = float(tot), "measured", f"county-counts/{cc['file']}" q_ev = f"St. Louis County turning-movement count {cc['file']}, {hour}-{int(hour[:2])+1:02d}:{hour[3:]} (L {mv['L']}, T {mv['T']}, R {mv['R']})" if tot: shares = {k: round(mv[k]/tot, 4) for k in ("L", "T", "R")}; sh_src, sh_ev = "measured", q_ev m = measured.get((lid, d)) if m: q, q_src, q_ev, q_ref = float(m["q_vph"]), "measured", m.get("note", "field count"), str(mfile.name) if m.get("L"): shares = {k: float(m[k]) for k in ("L", "T", "R")}; sh_src, sh_ev = "measured", m.get("note", "field count") apps[d] = dict(q_vph=round(q, 1), q_source=q_src, q_evidence=q_ev, q_ref=q_ref, lanes=lanes, lanes_source=lanes_src, s_vph=lanes*sa["sat_flow_per_lane_vph"], s_source=sa["tag_sat_flow"], turn_share=shares, turn_source=sh_src, turn_evidence=sh_ev) lights.append(dict(id=lid, name=c["name"], cross_street=c["cross_street"], lat=pos[0], lon=pos[1], approaches=apps, timing=county_timing(lid, pattern, asm["model"].get("lost_time_basis", "two_phase")))) sp = asm["speeds"]; links = [] for a, b in zip(lights, lights[1:]): L = haversine((a["lat"], a["lon"]), (b["lat"], b["lon"])) north = a["id"] in ("051",) # FPP->Forsyth link is north of Forsyth (30 mph); others 35 mph mph = sp["north_of_forsyth_mph"] if north else sp["south_of_forsyth_mph"] v = mph*0.44704 links.append(dict(**{"from": a["id"], "to": b["id"]}, length_m=round(L, 1), length_source="derived", length_evidence="straight-line distance between County signal coordinates", speed_mps=round(v, 2), speed_source=sp["tag"], tau_s=round(L/v, 2))) mo = asm["model"] Lmeas = [l["timing"]["L_s"] for l in lights if l["timing"]] L_global = round(sum(Lmeas)/len(Lmeas), 2) if len(Lmeas) == len(lights) else 2*sa["lost_time_per_phase_s"] L_src = "measured" if len(Lmeas) == len(lights) else sa["tag_lost_time"] L_ev = ("mean of per-light yellow+all-red (main + side) from County timing sheets; per-light values in lights[].timing.L_s" if L_src == "measured" else sa["evidence_lost_time"]) params = dict(L_s=L_global, L_source=L_src, L_evidence=L_ev, p=mo["p_platoon"], g_min_s=mo["g_min_s"], C_min_s=mo["C_min_s"], C_max_s=mo["C_max_s"], x_max=mo["x_max"], model_source=mo["tag"], yellow_s=sa["yellow_s"], all_red_s=sa["all_red_s"], sat_flow_per_lane_vph=sa["sat_flow_per_lane_vph"]) counts = {"measured": 0, "derived": 0, "assumed": 0} for l in lights: for a in l["approaches"].values(): counts[a["q_source"]] += 1 out = dict(meta=dict(name=cor["corridor"]["name"], peak=cor["corridor"]["peak"], built=datetime.datetime.now().isoformat(timespec="seconds"), version=("v2" if (measured or any(counts_by_light.values()) or Lmeas) else "v1"), peak_hour_factor=round(K, 4), peak_hour_factor_evidence=f"median of {len(ks)} County stations (PkHrVol/AWT)", approach_volume_sources=counts, design_hour=hour, timing_pattern=pattern, lights_with_measured_timing=[l["id"] for l in lights if l["timing"]]), lights=lights, links=links, params=params) print(f"corridor.json {out['meta']['version']}: {len(lights)} lights, K={K:.3f}, volumes {counts}") for l in lights: print(f" {l['id']} {l['name']:16s} " + " ".join(f"{d}:{l['approaches'][d]['q_vph']:7.1f}" for d in DIRS)) # effective saturation flows along the real phase sequence (see assumptions.toml [model].capacity_basis) if asm["model"].get("capacity_basis") == "signal_sequence": import nema rawt = json.load(open(MEAS/"county_timing.json")) if (MEAS/"county_timing.json").exists() else {} spl = sa["sat_flow_per_lane_vph"] for l in lights: T = l["timing"] if not T or l["id"] not in rawt: continue A = l["approaches"]; ph = {int(k): v for k, v in rawt[l["id"]]["phases"].items()} has_lt = {a for v in ph.values() for a in __import__("re").findall(r"(NB|SB|EB|WB)\s*LT", v["movement"])} nt = lambda a: (2 if a in ("NB", "SB") else max(A[a]["lanes"]-1, 1)) mv = lambda a, k: A[a]["q_vph"]*A[a]["turn_share"][k] def y_phase(n): prot, perm = nema._moves(ph[n]["movement"], has_lt); ys = [0.0] for a, ms in prot.items(): if ms == {"R"}: continue if ms == {"L"}: ys.append(mv(a, "L")/spl) elif "L" in ms: ys.append(A[a]["q_vph"]/(spl*(nt(a)+1))) else: ys.append((mv(a, "T")+mv(a, "R"))/(spl*nt(a))) return max(ys) sp = {int(k): v for k, v in T["splits"].items()} for block, dirs in ((T["main_phases"], ("NB", "SB")), ([n for n in sp if n not in T["main_phases"]], ("EB", "WB"))): Y = max(sum(y_phase(n) for n in block if (n <= 4) == (r == 1)) for r in (1, 2)) for d in dirs: A[d]["s_lanes_vph"] = A[d]["s_vph"] A[d]["s_vph"] = round(A[d]["q_vph"]/Y, 1) if Y > 0 else A[d]["s_vph"] A[d]["s_source"] = "derived" A[d]["s_evidence"] = (f"effective: q / Y_block, Y_block = {Y:.3f} = critical flow ratio of this block along the County " f"phase sequence (measured volumes, lanes; {spl} veh/h/lane)") # two-phase feasibility: sum over phases of max flow ratio y = q/s must leave room for lost time at C_max and x_max Lt, xm, Cmax = params["L_s"], params["x_max"], params["C_max_s"] for l in lights: A = l["approaches"] y_main = max(A[d]["q_vph"]/A[d]["s_vph"] for d in ("NB", "SB")); y_side = max(A[d]["q_vph"]/A[d]["s_vph"] for d in ("EB", "WB")) need = (y_main + y_side)/xm ok = need < 1 - Lt/Cmax print(f" feasibility {l['id']}: y_main={y_main:.2f} y_side={y_side:.2f} need {need:.2f} of cycle (< {1-Lt/Cmax:.2f}) {'OK' if ok else 'OVER CAPACITY'}") for k in links: print(f" link {k['from']}->{k['to']}: {k['length_m']} m, {k['speed_mps']} m/s, tau {k['tau_s']} s") _write(out) def _write(out): BUILD.mkdir(exist_ok=True) json.dump(out, open(BUILD/"corridor.json", "w"), indent=1) if __name__ == "__main__": sys.exit(main())