#!/usr/bin/env python3
"""Build developers.sqlite from quotes captured in this run. No inferred emails."""
import json
import re
import sqlite3
from pathlib import Path

ROOT = Path("/Users/hattr/Documents/wa-land-developers")
DB = ROOT / "developers.sqlite"
RET = "2026-10-19"
RAW = ROOT / "raw"

# tier, trading_name, entity_kind, locality, region, scale_note, source_url, source_title, evidence, extra_urls
# evidence must be text that appeared on the source page.
A = []

def add(tier, trading, kind, locality, region, scale, url, title, evidence, extra=None):
    evidence = re.sub(r"\s+", " ", evidence).strip()
    if not url or not evidence or not RET:
        raise SystemExit(f"missing source fields for {trading}")
    if "@" in evidence and tier != "skip":
        pass
    A.append({
        "tier": tier,
        "trading_name": trading,
        "legal_name": None,
        "entity_kind": kind,
        "locality": locality,
        "region_tag": region,
        "scale_note": scale,
        "source_url": url,
        "source_title": title,
        "evidence": evidence[:500],
        "extra": extra or [],
        "canonical": None,
    })

UDIA_ABOUT = "https://www.udiawa.com.au/about/"
UDIA_2026 = "https://www.udiawa.com.au/press-releases/2026-udia-wa-awards-celebrate-excellence-and-innovation-across-western-australias-development-industry/"
UDIA_2025 = "https://www.udiawa.com.au/press-releases/seniors-living-resort-wins-top-honours-at-30th-anniversary-awards-for-excellence/"
UDIA_2023 = "https://www.udiawa.com.au/press-releases/awards-showcase-range-of-stunning-projects-in-perth-south-west/"
UDIA_2020 = "https://www.udiawa.com.au/press-releases/focus-on-local-amenities-wins-top-honours-in-development-awards/"
UDIA_2019 = "https://www.udiawa.com.au/press-releases/diversity-and-innovation-the-big-winners-in-development-industry-awards-for-excellence/"
UDIA_2018 = "https://www.udiawa.com.au/press-releases/udia-award-winners-showcase-best-of-the-best-in-lifestyle-affordability-and-community/"
UDIA_2024 = "https://www.udiawa.com.au/press-releases/was-growing-housing-diversity-on-show-at-industry-awards/"
UDIA_HEID = "https://www.udiawa.com.au/hesperia-heidelberg-to-unlock-4000-homes/"
DEVWA = "https://developmentwa.com.au"
ZEN_P = "https://zenithenergy.com.au/peel/"
ZEN_E = "https://zenithenergy.com.au/eglinton-village-energy/"
ZEN_O = "https://zenithenergy.com.au/ocean-reef-marina/"

# --- Government land agency and the three Zenith seeds ---
add("A", "DevelopmentWA", "government land agency", "Perth", "swis",
    "hectares=1400; dwellings=3075. Statewide land agency. Latitude 32 is described as covering more than 1,400 hectares. The Village at Wellard is about 3,075 homes and Sienna Wood about 12,000 residents. Peel Business Park's first stage is part of 1,000 ha. Ocean Reef Marina is about 1,000 homes with a 4 MVA grid connection and a listed 5 MW capacity. These projects sit above the about 1,600-home, 75-hectare or 15 MVA Western Power planning lump. Former names on these pages include LandCorp and the Metropolitan Redevelopment Authority.",
    "https://developmentwa.com.au/our-work",
    "Our Work - DevelopmentWA",
    "Across Western Australia we develop land for living and land for working to create places to build our social and economic prosperity.",
    [
        "https://developmentwa.com.au/projects/industrial-and-commercial/latitude-32-industry-zone/overview",
        "https://developmentwa.com.au/projects/residential/the-village-at-wellard/overview",
        "https://developmentwa.com.au/projects/residential/sienna-wood/overview",
        "https://developmentwa.com.au/projects/industrial-and-commercial/ocean-reef-marina/overview",
        ZEN_P, ZEN_O,
        "https://www.cossillwebley.com.au/project/peel-business-park/",
        "https://www.cossillwebley.com.au/project/ocean-reef-marina-microgrid/",
        UDIA_2026,
    ])

add("A", "Cedar Woods", "land developer", "Eglinton", "swis",
    "dwellings=1300; solar_mw=4.4; connection_mw=5. Eglinton Village is about 1,300 homes with 4.4 MW of rooftop solar and a listed 5 MW capacity, under the 1,600-home lump and under a 10 MW connection and 10 MW solar ceiling.",
    ZEN_E, "Eglinton Village Energy - Zenith Energy",
    "Eglinton Village Energy (EVE) is the owner and operator of the microgrid for the Eglinton Village Microgrid, a residential estate developed by Cedar Woods Properties.",
    [
        "https://www.cossillwebley.com.au/project/eglinton-village/",
        "https://developmentwa.com.au/projects/residential/harrisdale-green/overview",
        UDIA_2026, UDIA_2020,
    ])

add("A", "Stockland", "land developer", "Eglinton", "swis",
    "dwellings=2686. Amberton is in the public ULDO as about 2,686 dwellings and Sienna Wood as about 2,019 dwellings; DevelopmentWA says Sienna Wood will house almost 12,000 residents. That is above the about 1,600-home planning lump.",
    "https://www.cossillwebley.com.au/project/amberton-beach/",
    "Amberton Beach - Cossill & Webley",
    "Client: Stockland Location: Eglinton, WA",
    [
        "https://developmentwa.com.au/projects/residential/sienna-wood/overview",
        "https://developmentwa.com.au/projects/residential/alkimos-beach/overview",
        "https://developmentwa.com.au/projects/residential/alkimos-vista/overview",
        UDIA_2026, UDIA_2020, UDIA_2024,
        "https://www.udiawa.com.au/stockland-takes-top-prize/",
    ])

add("A", "Satterley Property Group", "land developer", "Eglinton", "swis",
    "dwellings=1688. Allara is a partnering arrangement with Satterley. Public ULDO lists Allara at about 1,688 dwellings, around the 1,600-home planning lump. Ellenbrook is also named as a Satterley project.",
    "https://developmentwa.com.au/projects/residential/allara/overview",
    "Allara, Eglinton - Overview - DevelopmentWA",
    "Allara in Eglinton is a partnering arrangement with Satterley Property Group.",
    [
        "https://developmentwa.com.au/our-work",
        "https://developmentwa.com.au/projects/residential/cassia/overview",
        "https://www.cossillwebley.com.au/project/ellenbrook/",
        UDIA_ABOUT, UDIA_2025, UDIA_2024, UDIA_2019,
    ])

add("A", "Peet Limited", "land developer", "Wellard", "swis",
    "dwellings=3075. The Village at Wellard is about 3,075 homes and Golden Bay about 2,200 homes, both above the 1,600-home lump. Brabham is delivered in partnership with PEET.",
    "https://developmentwa.com.au/projects/residential/the-village-at-wellard/overview",
    "The Village at Wellard - DevelopmentWA",
    "The Village at Wellard, developed in partnership with Peet Limited, is expected to deliver around 3,075 homes upon completion.",
    [
        "https://developmentwa.com.au/projects/residential/golden-bay/overview",
        "https://developmentwa.com.au/projects/residential/brabham/overview",
        "https://www.cossillwebley.com.au/project/brabham-estate/",
        "https://www.cossillwebley.com.au/project/golden-bay/",
        UDIA_2025, UDIA_2018, UDIA_ABOUT,
    ])

add("A", "Peet Golden Bay Pty Ltd", "land developer", "Golden Bay", "swis",
    "dwellings=2200. About 2,200 homes, above the 1,600-home planning lump. Separate legal name from Peet Limited on this page.",
    "https://developmentwa.com.au/projects/residential/golden-bay/overview",
    "Golden Bay - DevelopmentWA",
    "The Golden Bay estate, developed in partnership with Peet Golden Bay Pty Ltd, is expected to deliver around 2200 homes upon completion.",
    [])

add("A", "Lendlease", "land developer", "Alkimos", "swis",
    "dwellings=unknown. Master-planned coastal community with a major regional centre. Dwelling count is not stated on this page. Alkimos-scale coastal communities in the public ULDO run into the thousands of dwellings, which would sit above a sub-10 MW single connection.",
    "https://developmentwa.com.au/our-work",
    "Our Work - DevelopmentWA",
    "Alkimos , where we have partnered with Lendlease to create a master-planned, coastal community with a major regional centre and coastal village",
    [])

add("A", "Hesperia", "land developer", "Ellenbrook", "swis",
    "dwellings=4000. A proposal to unlock land west of Ellenbrook for about 4,000 homes, above the 1,600-home lump. Also named on the UDIA WA Council.",
    UDIA_HEID, "Hesperia, Heidelberg to unlock 4,000 homes - UDIA WA",
    "Hesperia and Heidelberg Materials are progressing a proposal to unlock land for residential development in Ellenbrook",
    [
        UDIA_ABOUT, UDIA_2023,
        "https://developmentwa.com.au/projects/residential/murdoch-health-and-knowledge-precinct/overview",
    ])

add("A", "Heidelberg Materials", "land developer", "Ellenbrook", "swis",
    "dwellings=4000. Named with Hesperia on a proposal for about 4,000 homes on land that is subject to a sand mining lease. Above the 1,600-home lump. The full Business News article is not on this page.",
    UDIA_HEID, "Hesperia, Heidelberg to unlock 4,000 homes - UDIA WA",
    "Ben Lisle-led Hesperia has entered a consortium with Heidelberg to unlock a significant parcel of land west of Ellenbrook for housing, which is currently subject to Heidelberg’s sand mining lease.",
    [])

add("A", "Parcel Property", "land developer", "Perth", "swis",
    "dwellings=469. Beenyup Grove is to comprise 469 lots, under the 1,600-home lump and likely under a 10 MW connection. Also an infill developer (NOMA, Evolve Duncraig).",
    "https://developmentwa.com.au/projects/residential/beenyup-grove/overview",
    "Beenyup Grove - DevelopmentWA",
    "Beenyup Grove is delivered in partnership with Parcel Property Group.",
    [UDIA_ABOUT, UDIA_2023, UDIA_2020, UDIA_2019])

add("A", "Okeland Communities", "land developer", "Perth", "swis",
    "dwellings=70000. The 2026 awards say the organisation has contributed to unlocking more than 70,000 homes across Australia. That portfolio is far above a single 1,600-home or sub-10 MW connection. WA project sizes are not stated on this page.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "Okeland Communities was named Team of the Year, recognising its family-led approach, collaborative culture and contribution to unlocking more than 70,000 homes across Australia.",
    [UDIA_ABOUT])

add("A", "Mirvac", "land developer", "Henley Brook", "swis",
    "dwellings=unknown. Henley Brook and Illuma Private Estate are named as Mirvac developments. Lot count is not in the award lines. ULDO lists Brooklands Private Estate at Henley Brook at about 2,993 dwellings; that estate name is not tied to Mirvac on the award page.",
    UDIA_2023, "Awards showcase range of stunning projects in Perth & South West",
    "Henley Brook by Mirvac",
    [UDIA_ABOUT, UDIA_2020])

add("A", "LWP Group", "land developer", "Alkimos", "swis",
    "dwellings=4000. Banksia Grove is about 4,000 homes on 338 hectares, above both the 1,600-home and 75-hectare lumps. LWP is also named on Trinity at Alkimos and the Verge micro-lot precinct.",
    "https://developmentwa.com.au/projects/residential/mahala/overview",
    "Mahala - DevelopmentWA",
    "Mahala is delivered in partnership with LWP Group.",
    [
        "https://developmentwa.com.au/projects/residential/banksia-grove/overview",
        UDIA_2018,
    ])

add("A", "Banksia Grove Development Nominees Pty Ltd", "land developer", "Banksia Grove", "swis",
    "dwellings=4000; hectares=338. About 4,000 homes on 338 hectares, above the 1,600-home and 75-hectare planning lumps.",
    "https://developmentwa.com.au/projects/residential/banksia-grove/overview",
    "Banksia Grove - DevelopmentWA",
    "Banksia Grove is delivered in partnership with Banksia Grove Development Nominees Pty Ltd.",
    [])

add("A", "Wolfdene", "land developer", "South Yunderup", "swis",
    "dwellings=unknown. Austin Lakes is described as a major masterplanned community in the Peel region. Lot count is not stated. A Peel masterplanned estate can reach the 1,600-home lump; this page does not give the number.",
    "https://www.cossillwebley.com.au/project/austin-lakes/",
    "Austin Lakes - Cossill & Webley",
    "Originally engaged by Satterley and continuing under current developer Wolfdene, Cossill & Webley has played a key role in shaping the civil engineering outcomes for this growing community.",
    [UDIA_ABOUT])

add("A", "Perron Developments", "land developer", "Witchcliffe", "swis",
    "dwellings=unknown. Witchcliffe Ecovillage and Brabham Estate name Perron as a partner. Lot count is not on the award line.",
    UDIA_2023, "Awards showcase range of stunning projects in Perth & South West",
    "Witchcliffe Ecovillage by Sustainable Settlements Pty Ltd and Perron Developments Pty Ltd",
    [UDIA_2025, "https://www.cossillwebley.com.au/project/witchcliffe-ecovillage/"])

add("A", "Sustainable Settlements Pty Ltd", "land developer", "Witchcliffe", "swis",
    "dwellings=unknown. Named with Perron Developments as the Witchcliffe Ecovillage developer. Scale is not stated on the award line.",
    UDIA_2023, "Awards showcase range of stunning projects in Perth & South West",
    "Witchcliffe Ecovillage by Sustainable Settlements Pty Ltd and Perron Developments Pty Ltd",
    ["https://www.cossillwebley.com.au/project/witchcliffe-ecovillage/"])

add("A", "Finbar", "apartment developer", "Perth", "swis",
    "dwellings=unknown. Apartment projects Civic Heart, Sabina and others are named. A single apartment building is often under a 10 MW connection; the page does not state a load.",
    UDIA_2025, "2025 UDIA WA Awards for Excellence",
    "Civic Heart by Finbar",
    [UDIA_ABOUT, UDIA_2020, UDIA_2024])

add("A", "Blackburne", "apartment developer", "Subiaco", "swis",
    "dwellings=unknown. ONE Subiaco is described as bringing about 500 new residents. That is under the 1,600-home lump.",
    "https://www.udiawa.com.au/blackburne-property-groups-one-subiaco-takes-out-top-gong-at-urban-development-institute-of-australia-awards/",
    "Blackburne Property Group’s ONE Subiaco - UDIA WA",
    "ONE Subiaco by Blackburne Property Group has taken out the top gong at the Urban Development Institute of Australia (UDIA WA) Awards for Excellence 2023.",
    [UDIA_2026, UDIA_2025, UDIA_2020, UDIA_2018])

add("A", "Edge Visionary Living", "apartment developer", "Perth", "swis",
    "dwellings=unknown. Riviere Residences and other apartment projects are named. Load is not stated.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "Rivière Residences by Edge Visionary Living winning the Apartments (High-Rise) category.",
    [
        "https://developmentwa.com.au/projects/residential/burt-street/overview",
        UDIA_2020,
    ])

add("A", "Celsius Property Group", "apartment developer", "Subiaco", "swis",
    "dwellings=unknown. Elysian Subiaco and The Vic Quarter are named. Load is not stated.",
    UDIA_2025, "2025 UDIA WA Awards for Excellence",
    "Elysian Subiaco by Celsius Developments",
    [UDIA_ABOUT, UDIA_2020])

add("A", "Yolk Property Group", "apartment developer", "White Gum Valley", "swis",
    "dwellings=unknown. Evermore Apartments at White Gum Valley won an affordable-development award. Load is not stated.",
    UDIA_2018, "2018 UDIA WA Award winners",
    "Evermore Apartments WGV entered by Yolk Property Group",
    [UDIA_ABOUT])

add("A", "Forge Developments", "infill developer", "Nedlands", "swis",
    "dwellings=3. Kingsway Townhouses is three townhomes, far under the 1,600-home lump and under a 10 MW connection.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "Kingsway Townhouses by Forge Developments won the Boutique Development category for delivering three generously sized premium townhomes on a landmark corner site in Nedlands.",
    [])

add("A", "Willing", "infill developer", "Mount Lawley", "swis",
    "dwellings=unknown. West Residences is one apartment building in Mount Lawley. Under the 1,600-home lump.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "The industry’s highest honour, the Russel Perry Award for Urban Development Excellence, was presented to West Residences by Willing.",
    [])

add("A", "DevelopWise", "infill developer", "Mount Pleasant", "swis",
    "dwellings=unknown. Oasis Mount Pleasant is one apartment building. Load is not stated.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "Oasis Mount Pleasant by DevelopWise won the Design category",
    [])

add("A", "Linic Group", "infill developer", "Perth", "swis",
    "dwellings=28. Cygnus Parade is 28 townhomes, apartments and lofts, under the 1,600-home lump.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "The Medium Density category winner was yet another infill project, Cygnus Parade by Linic Group. Delivering 28 townhomes, apartments and lofts on a compact site",
    [])

add("A", "The HMO Property Co", "infill developer", "Bayswater", "swis",
    "dwellings=unknown. A co-living building at Jacqueline Street, Bayswater. Load is not stated. Under estate scale.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "The Affordable Housing category was won by The HMO Coliving Model at Jacqueline Street, Bayswater by The HMO Property Co.",
    [])

add("A", "Exal Group", "student housing developer", "Bentley", "swis",
    "dwellings=916. Yugo Perth University Campus is up to 916 student beds. That is under 1,600 homes but is a large single electrical load; megawatts are not stated.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "Yugo Perth University Campus by Exal Group was recognised for transforming an underutilised suburban area into an extension of the Greater Curtin Precinct and delivering up to 916 student beds across two stages.",
    [])

add("A", "Foundation Housing Limited", "community housing developer", "Kenwick", "swis",
    "dwellings=34. Named for a 34-apartment social housing building in East Fremantle and for specialist disability homes in Kenwick. Both are under the 1,600-home lump and under a 10 MW connection.",
    "https://developmentwa.com.au/projects/residential/housing/developmentwa-supported",
    "DevelopmentWA supported housing",
    "Foundation Housing Limited is the preferred proponent to construct, own and operate the new 34 social apartments, consisting of 19 one-bedroom and 15 two-bedroom dwellings.",
    [UDIA_2026])

add("A", "Community Housing Limited", "community housing developer", "Subiaco", "swis",
    "dwellings=447. A 447-dwelling community housing development is under the 1,600-home lump. A single inner-city building of this size can still be a multi-megawatt connection; megawatts are not stated.",
    "https://developmentwa.com.au/projects/residential/housing/developmentwa-supported",
    "DevelopmentWA supported housing",
    "Community Housing Limited (CHL) is the preferred proponent to construct, own and operate the new 447-dwelling community housing development, consisting of 300 affordable homes and 147 social homes.",
    [])

add("A", "Providence Lifestyle Group", "land-lease community developer", "Piara Waters", "swis",
    "dwellings=unknown. Piara Waters Lifestyle Village won the 2025 top award. Village size is not stated on this page.",
    UDIA_2025, "2025 UDIA WA Awards for Excellence",
    "Piara Waters Lifestyle Village by Providence Lifestyle Group",
    [UDIA_ABOUT])

add("A", "Department of Communities", "government housing developer", "Perth", "swis",
    "dwellings=unknown. Named as a development partner on Banksia Grove, SugarGum Estate, Dalyellup Beach and the Verge micro-lot precinct. Banksia Grove as a whole is about 4,000 homes; this page does not split the department's share.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "Banksia Grove Development by LWP Group, Department of Communities and Banksia Grove Development Nominees",
    [UDIA_2019, UDIA_2018])

add("A", "Perth Airport", "airport land developer", "Perth Airport", "swis",
    "hectares=223. Cossill & Webley describes subdivision planning for an industrial precinct at Perth Airport North. Public ULDO lists Airport North at about 223 hectares, above the 75-hectare lump. The ULDO site name is not repeated on the Cossill & Webley page.",
    "https://www.cossillwebley.com.au/project/perth-airport/",
    "Perth Airport - Cossill & Webley",
    "Cossill & Webley played a key role in the planning and engineering design of a proposed industrial precinct at Perth Airport North, subject to regulatory approvals.",
    [UDIA_2025])

add("A", "First One Developments", "industrial land developer", "Neerabup", "swis",
    "hectares=unknown. Global Business Park is an industrial strata complex in Neerabup. Area and electrical demand are not stated on this page. Neerabup industrial land in the public ULDO is hundreds of hectares across several sites.",
    "https://www.cossillwebley.com.au/project/global-business-park/",
    "Global Business Park - Cossill & Webley",
    "Client: First One Developments",
    [])

add("A", "Aigle Royal Group", "industrial and mixed-use developer", "Casuarina", "swis",
    "hectares=60. Samara Casuarina is a 60-hectare masterplanned development of large-format retail, commercial and residential precincts, near the 75-hectare planning lump. A Costco is named as an end user.",
    "https://www.cossillwebley.com.au/project/samara-casuarina/",
    "Samara Casuarina - Cossill & Webley",
    "Samara Casuarina is a 60-hectare masterplanned development comprising of large format retail, commercial and residential precincts.",
    [])

add("A", "Golden Sedayu", "infill developer", "Burswood", "swis",
    "dwellings=unknown. Client of the Burswood Point infill project. Dwelling count is not on the Cossill & Webley page. Public ULDO lists Burswood Point at about 3,000 and 1,500 dwellings on two polygons; that page does not name Golden Sedayu.",
    "https://www.cossillwebley.com.au/project/burswood-point/",
    "Burswood Point - Cossill & Webley",
    "Client: Golden Sedayu Location: Belmont, WA",
    [])

add("A", "George Weston Foods", "landowner client of a residential redevelopment", "Spearwood", "swis",
    "dwellings=unknown. Named as the client for turning a former processing factory into a residential development (Eliza Ponds). Lot count is not stated.",
    "https://www.cossillwebley.com.au/project/eliza-ponds/",
    "Eliza Ponds - Cossill & Webley",
    "Client: George Weston Foods",
    [])

add("A", "Morella", "land developer", "Ellenbrook", "swis",
    "dwellings=unknown. Ellenbrook is delivered in partnership with Morella. Estate size is not stated on this overview.",
    "https://developmentwa.com.au/projects/residential/ellenbrook/overview",
    "Ellenbrook - DevelopmentWA",
    "Ellenbrook delivered in partnership with Morella.",
    [])

add("A", "Australian Capital Equity", "precinct proponent", "East Perth", "swis",
    "dwellings=unknown. Named as a preferred proponent for the East Perth Power Station precinct. The same timeline says the development deed later expired. Scale is not stated.",
    "https://developmentwa.com.au/projects/residential/east-perth-power-station/development-timeline",
    "East Perth Power Station timeline - DevelopmentWA",
    "Australian Capital Equity and Minderoo Group (Koomba Kalark) named preferred proponents to progress the revitalisation of the East Perth Power Station precinct.",
    [])

add("A", "Minderoo Group", "precinct proponent", "East Perth", "swis",
    "dwellings=unknown. Named with Australian Capital Equity as preferred proponent for East Perth Power Station. Scale is not stated.",
    "https://developmentwa.com.au/projects/residential/east-perth-power-station/development-timeline",
    "East Perth Power Station timeline - DevelopmentWA",
    "Australian Capital Equity and Minderoo Group (Koomba Kalark) named preferred proponents to progress the revitalisation of the East Perth Power Station precinct.",
    [])

add("B", "Aegis Health", "healthcare occupier and development partner", "Murdoch", "swis",
    "dwellings=unknown. Murdoch Square is a mixed-use healthcare precinct developed by Hesperia and Aegis Health. Electrical load is not stated. A healthcare precinct can be a large connection; this page does not give MVA.",
    "https://developmentwa.com.au/projects/residential/murdoch-health-and-knowledge-precinct/overview",
    "Murdoch Health and Knowledge Precinct - DevelopmentWA",
    "Murdoch Square is Western Australia’s newest mixed-use healthcare precinct, developed by Hesperia and Aegis Health.",
    [])

add("A", "Stirling Capital", "apartment developer", "Mount Pleasant", "swis",
    "dwellings=unknown. Cirque Mount Pleasant is one apartment building aimed at downsizers. Load is not stated.",
    UDIA_2019, "2019 UDIA WA Awards for Excellence",
    "Cirque Mount Pleasant, a luxurious apartment building by Stirling Capital, has successfully tapped into the growing downsizer market",
    [])

add("A", "Westbridge Urban", "infill developer", "Dalkeith", "swis",
    "dwellings=unknown. Dalkeith Terraces is a townhouse project after a rezoning. Load is not stated. Under estate scale.",
    UDIA_2024, "2024 UDIA WA Awards for Excellence",
    "Dalkeith Terraces by Westbridge Urban won in the Boutique Development category",
    [])

add("A", "Megara Property", "apartment developer", "Sorrento", "swis",
    "dwellings=unknown. Ora Sorrento is named as an award winner. Load is not stated.",
    UDIA_2023, "Awards showcase range of stunning projects in Perth & South West",
    "Ora Sorrento by Megara Property",
    [])

add("A", "Now Living", "infill developer", "Girrawheen", "swis",
    "dwellings=unknown. Verge micro-lot precinct, with LWP and the Department of Communities. Lot count is not in the award line.",
    UDIA_2018, "2018 UDIA WA Award winners",
    "Verge Micro Lot Precinct entered by Now Living, LWP Property & Department of Communities",
    [UDIA_2019])

add("A", "BGC Development", "land and mixed-use developer", "Perth", "swis",
    "dwellings=unknown. Hibernian Place and the Westin Perth is a mixed-use city project. Load is not stated. A city hotel and plaza is under the 1,600-home lump.",
    UDIA_2018, "2018 UDIA WA Award winners",
    "Hibernian Place + The Westin Perth entered by BGC Development",
    [])

add("A", "Defence Housing Australia", "housing developer", "Fremantle", "swis",
    "dwellings=unknown. Liv Apartments in Fremantle. Load is not stated.",
    UDIA_2018, "2018 UDIA WA Award winners",
    "Liv Apartments entered by Defence Housing Australia",
    [])

add("A", "Noahs Rosehill Waters Pty Ltd", "land developer", "South Guildford", "swis",
    "dwellings=unknown. Rosehill Waters won an environmental excellence award. Lot count is not stated.",
    UDIA_2018, "2018 UDIA WA Award winners",
    "Rosehill Waters entered by Noahs Rosehill Waters Pty Ltd",
    [])

add("A", "Building Development Group", "lifestyle village developer", "Margaret River", "swis",
    "dwellings=unknown. Margaret River Lifestyle Village. Size is not stated. Margaret River is on the South West Interconnected System.",
    UDIA_2018, "2018 UDIA WA Award winners",
    "Margaret River Lifestyle Village entered by Building Development Group",
    [])

add("A", "Norup & Wilson", "infill developer", "Perth", "swis",
    "dwellings=unknown. The Precinct received a judges award in 2020. Size is not stated.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "The Precinct by Norup & Wilson",
    [])

add("A", "Giorgi", "infill developer", "Perth", "swis",
    "dwellings=unknown. Echelon won the 2020 medium-density category. Size is not stated.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "Echelon by Giorgi won in the Medium Density category",
    [])

add("A", "Fini Developments", "infill developer", "Perth", "swis",
    "dwellings=unknown. Henley On Park is a 2020 medium-density winner. Size is not stated.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "Henley On Park by Fini Developments",
    [])

add("A", "Fowler Group Properties", "apartment developer", "Perth", "swis",
    "dwellings=unknown. The Vic Quarter, with Celsius Property Group. Size is not stated.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "The Vic Quarter by Fowler Group Properties and Celsius Property Group",
    [])

add("A", "Far East Consortium", "apartment developer", "Perth", "swis",
    "dwellings=unknown. The Towers at Elizabeth Quay. An interstate firm with a Perth project. Load is not stated. A CBD tower can be a multi-megawatt building; this page does not give MVA.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "The Towers at Elizabeth Quay by Far East Consortium and Cottee Parker Architects",
    [])

add("A", "Nicheliving", "house-and-land developer", "Hammond Park", "swis",
    "dwellings=unknown. Nicheliving Hammond Park won an affordable-development award. Also a volume builder. Lot count is not stated.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "Nicheliving Hammond Park by Nicheliving",
    [])

add("A", "Adventist Care", "seniors housing developer", "Rossmoyne", "swis",
    "dwellings=unknown. Australis at Rossmoyne. Bed count is not stated.",
    UDIA_2019, "2019 UDIA WA Awards for Excellence",
    "Australis at Rossmoyne by Hames Sharley and Adventist Care",
    [])

add("A", "Hall & Prior", "aged-care developer", "Perth", "swis",
    "dwellings=unknown. Karingal Health and Aged Care Community. Bed count and electrical load are not stated.",
    UDIA_2020, "2020 UDIA WA Awards for Excellence",
    "Karingal Health and Aged Care Community by Hall & Prior and HASSELL",
    [])

add("A", "Sirona Urban", "developer", "Perth", "swis",
    "dwellings=unknown. Angela Wright Bennett Centre, with Ruah Community Services. Size is not stated.",
    UDIA_2025, "2025 UDIA WA Awards for Excellence",
    "Angela Wright Bennett Centre by Sirona Urban & Ruah Community Services",
    [])

add("A", "Town of Victoria Park", "local government developer", "Victoria Park", "swis",
    "hectares=unknown. Elizabeth Baillie redevelopment of a long-vacant site into a community and commercial destination. Area and load are not stated. Under a metro-wide 1,600-home lump.",
    UDIA_2026, "2026 UDIA WA Awards for Excellence",
    "The Social and Community Infrastructure category went to the Town of Victoria Park for the beautiful Elizabeth Baillie Redevelopment Project that has transformed a long-vacant site into a thriving community and commercial destination.",
    [])

add("A", "City of Stirling", "local government developer", "Hamersley", "swis",
    "dwellings=unknown. Hamersley Golf Course Redevelopment is named as a 2025 project winner. Lot count is not in the winners list line.",
    UDIA_2025, "2025 UDIA WA Awards for Excellence",
    "Hamersley Golf Course Redevelopment by City of Stirling",
    [])

add("A", "Shire of Broome", "local government", "Broome", "horizon",
    "dwellings=unknown. Chinatown revitalisation. Broome is supplied by Horizon Power, not the SWIS. Load is not stated.",
    UDIA_2019, "2019 UDIA WA Awards for Excellence",
    "Another regional project recognised in the awards program was the impressive Chinatown Revitalisation by the Shire of Broome.",
    [])

add("A", "Hamersley Iron", "historical town developer", "Dampier", "horizon",
    "dwellings=unknown. The page says Hamersley Iron, a subsidiary of Rio Tinto, developed Dampier as a port town with housing. This is not an off-grid mine power station. Dampier is in the Horizon Power area. The page is historical and does not state a current connection size.",
    "https://developmentwa.com.au/projects/residential/pilbara-vernacular/dampier",
    "Dampier - DevelopmentWA",
    "It was developed by Hamersley Iron (a subsidiary of Rio Tinto) as a port facility and to provide housing fo",
    [])

add("A", "Smiths 2014 Pty Ltd", "tourism and residential proponent", "Yallingup", "swis",
    "dwellings=unknown. Named as the proponent of the Smiths Beach proposal at Yallingup. Yallingup is on the South West Interconnected System. Dwelling count is not in this determination line.",
    "https://www.epa.wa.gov.au/sites/default/files/Extract_of_determination/831%20Chairmans%20Determination.pdf",
    "Chairman’s determination, Smiths Beach",
    "Proponent: Smiths 2014 Pty Ltd",
    [])

add("B", "Hanson Construction Materials Pty Ltd", "industrial occupier", "Perth", "swis",
    "dwellings=unknown. A seven-year extension of an existing concrete batching plant in Perth, not a new estate. Megawatts are not stated. Kept as an industrial occupier.",
    "https://www.planning.wa.gov.au/docs/default-source/sdau/27-jun-2024---wapc-media-statement---concrete-batching-plants-determination.pdf?sfvrsn=f08e620c_2",
    "WAPC media statement, concrete batching plants",
    "Applicants Hanson Construction Materials Pty Ltd and Holcim Australia Pty Ltd had applied for a seven-year extension to their operations in Edward Street and Claisebrook Road",
    [])

add("B", "Holcim Australia Pty Ltd", "industrial occupier", "Perth", "swis",
    "dwellings=unknown. Named with Hanson on a seven-year extension of existing batching plants. Megawatts are not stated.",
    "https://www.planning.wa.gov.au/docs/default-source/sdau/27-jun-2024---wapc-media-statement---concrete-batching-plants-determination.pdf?sfvrsn=f08e620c_2",
    "WAPC media statement, concrete batching plants",
    "Applicants Hanson Construction Materials Pty Ltd and Holcim Australia Pty Ltd had applied for a seven-year extension to their operations in Edward Street and Claisebrook Road",
    [])

add("B", "Scentre Group", "shopping centre owner", "Joondalup", "swis",
    "dwellings=unknown. Named as applicant and, with RE1 Limited, as owner on a City of Joondalup DAP agenda. A shopping centre is a large existing load; this cover line does not state MVA.",
    "https://www.planning.wa.gov.au/docs/default-source/daps/20221024-agenda-no-204-city-of-joondalup.pdf?sfvrsn=e9f9a7c6_5",
    "DAP agenda, City of Joondalup, 24 October 2022",
    "Applicant: Scentre Group C/O Urbis",
    [])

# Fix truncated Hamersley evidence by reading the page if present.
def fix_hamersley():
    from bs4 import BeautifulSoup
    import sys
    sys.path.insert(0, str(ROOT / "work"))
    from fetch import cache_path
    url = "https://developmentwa.com.au/projects/residential/pilbara-vernacular/dampier"
    cp = cache_path(url)
    if not cp.exists():
        return
    text = re.sub(r"\s+", " ", BeautifulSoup(cp.read_bytes(), "lxml").get_text(" ", strip=True))
    i = text.lower().find("hamersley")
    if i < 0:
        return
    quote = text[max(0, i - 40):i + 220].strip()
    for row in A:
        if row["trading_name"] == "Hamersley Iron":
            row["evidence"] = quote[:500]

fix_hamersley()

# Northerly and similar council-only names: the about page lists "Michael Lawson" then "Northerly Group" on separate lines.
# Confirm the quote exists.
def confirm_about_quotes():
    from bs4 import BeautifulSoup
    import sys
    sys.path.insert(0, str(ROOT / "work"))
    from fetch import cache_path
    cp = cache_path(UDIA_ABOUT)
    if not cp.exists():
        return
    text = BeautifulSoup(cp.read_bytes(), "lxml").get_text("\n", strip=True)
    flat = re.sub(r"\s+", " ", text)
    for row in A:
        if row["source_url"] != UDIA_ABOUT:
            continue
        # evidence is two adjacent names; check both tokens exist
        parts = row["evidence"].split()
        # require the organisation token is present
        org = row["trading_name"]
        if org.lower() not in flat.lower():
            raise SystemExit(f"about page missing {org}")

confirm_about_quotes()


def horizon_or_swis(address: str) -> str:
    addr = address or ""
    if re.search(r"\b(QLD|NSW|VIC|SA|TAS|NT|ACT)\s+\d{4}\b", addr) and not re.search(r"\bWA\b", addr):
        return "unknown"
    pcs = [int(x) for x in re.findall(r"\b(6\d{3})\b", addr)]
    if not pcs:
        return "unknown"
    pc = pcs[-1]
    if 6430 <= pc <= 6450 or 6640 <= pc <= 6655 or pc >= 6700:
        return "horizon"
    if 6000 <= pc <= 6999:
        return "swis"
    return "unknown"


_TAIL = {"Park", "Beach", "Hill", "Hills", "Grove", "Lakes", "Waters", "Springs", "Crossing", "Vale", "Brook", "Ridge", "Views", "Heights", "Downs", "Loop"}


def locality_from_address(address: str):
    addr = address or ""
    comma = re.search(r",\s*([A-Z][A-Z]+(?:\s+[A-Z][A-Z]+){0,2})\b", addr)
    tail = re.search(r"\b([A-Z][A-Z'’]+(?:\s+[A-Z][A-Z'’]+){0,2})\s+WA\s+\d{4}\b", addr)
    bad = {"Register", "Section", "Current", "Building", "Services", "Notes", "Page", "Practitioners", "Contractors", "John", "Ryan"}
    tail_loc = tail.group(1).title().replace("’", "'") if tail else None
    comma_loc = comma.group(1).title() if comma else None
    if tail_loc and any(w in bad for w in tail_loc.split()):
        tail_loc = None
    def clean(s):
        if not s:
            return None
        parts = [p for p in s.replace("’", "'").split() if p.lower() != "wa"]
        if not parts:
            return None
        return " ".join(parts)

    if tail_loc and tail_loc.split()[-1] in _TAIL and comma_loc and not comma_loc.endswith(tail_loc.split()[-1]):
        return clean(f"{comma_loc} {tail_loc.split()[-1]}")
    if tail_loc and tail_loc not in _TAIL:
        return clean(tail_loc)
    if comma_loc and not any(w in bad for w in comma_loc.split()):
        return clean(comma_loc)
    return clean(tail_loc)


def load_builders(con):
    text = (RAW / "BuilderRegister.txt").read_text(errors="replace")
    i1 = text.find("SECTION 1")
    i2 = text.find("SECTION 2")
    sec = text[i1:i2]
    url = "https://contenthub.demirs.wa.gov.au/downloads/cals/BuilderRegister.pdf"
    title = "Register of Building Contractors and Practitioners as at 18/10/2026"
    n = 0
    cur_rows = []
    lines = sec.splitlines()
    for i, line in enumerate(lines):
        m = re.match(r"Current\s+(BC\d+)\s{2,}(.+?)\s{2,}(.*)", line)
        if not m:
            continue
        reg, name, rest = m.group(1), m.group(2).strip(), m.group(3).strip()
        nxt = " ".join(lines[i + 1].split()) if i + 1 < len(lines) else ""
        if nxt and not nxt.startswith("Current") and len(nxt) <= 40 and "REGISTER" not in nxt and "SECTION" not in nxt:
            if re.fullmatch(r"\d{4}", nxt) or re.search(r"\bWA\b", nxt):
                rest = rest + " " + nxt
        cur_rows.append({"reg": reg, "name": name, "address": rest, "line": re.sub(r"\s+", " ", line.strip())})
    for rec in cur_rows:
        addr = re.sub(r"\s+", " ", rec["address"])
        # locality: last ALL-CAPS suburb before state if present, else leave a short address
        locality = locality_from_address(addr)
        region = horizon_or_swis(addr)
        evidence = f"Current {rec['reg']} {rec['name']}"
        if locality:
            evidence += f", address includes {locality}"
        evidence = evidence[:240]
        con.execute(
            """INSERT INTO developers
            (tier, legal_name, trading_name, abn, entity_kind, contact_name, contact_role,
             public_phone, public_email, website, locality, region_tag, scale_note,
             source_url, source_title, evidence, retrieved_at, duplicate_of)
            VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL)""",
            ("C", rec["name"], rec["name"], None, "registered building contractor",
             None, None, None, None, None, locality, region,
             f"Registration {rec['reg']}. Tier C building contractor from the public register. Not scored against the sub-10 MW connection ceiling.",
             url, title, evidence, RET),
        )
        n += 1
    return n


def dap_rows():
    path = RAW / "dap-small-names.json"
    if not path.exists():
        return
    rows = json.loads(path.read_text())
    skip_owner = re.compile(
        r"value of development|various land|water corp|water corporation|state of wa|aboriginal lands trust|wind farm|project haber|otr \d",
        re.I,
    )
    company = re.compile(r"pty|ltd|limited|inc\b|group|holdings|college|property|developments|homes|management|scentre|peet", re.I)
    seen = set()
    for rec in rows:
        url = rec["url"]
        owners = []
        for line in rec["found"]:
            if line.lower().startswith("owner:"):
                owners.append(line.split(":", 1)[1].strip())
        for owner in owners:
            owner = owner.strip(" ,./")
            if skip_owner.search(owner):
                continue
            if not company.search(owner):
                continue
            key = re.sub(r"[\s,./]+", " ", owner).strip().lower()
            if key in seen:
                # still keep extra url on first later via source_urls; skip duplicate insert here
                continue
            seen.add(key)
            # Peet and Scentre already have richer rows; attach URL only
            if re.search(r"^peet limited$", owner, re.I):
                for row in A:
                    if row["trading_name"] == "Peet Limited":
                        row["extra"].append(url)
                continue
            tier = "B" if re.search(r"scentre|college", owner, re.I) else "A"
            kind = "shopping centre owner" if "scentre" in owner.lower() else "DAP landowner company"
            if "college" in owner.lower():
                kind = "institutional landowner"
                tier = "B"
            add(tier, owner, kind, None, "swis",
                "dwellings=unknown. Named as owner on one publicly downloaded DAP agenda under 15 MB. The cover lines used here do not state homes, hectares or megawatts. Kept because the company is a landowner on a development application. Not removed for missing the sub-10 MW test.",
                url, "DAP agenda",
                f"Owner: {owner}",
                [])


def verify_evidence():
    """Every A/B evidence string must occur in the fetched source, after whitespace collapse."""
    import sys
    from bs4 import BeautifulSoup
    sys.path.insert(0, str(ROOT / "work"))
    from fetch import cache_path
    import subprocess
    article_by_link = {}
    for f in (RAW / "udia-articles").glob("*.json"):
        rec = json.loads(f.read_text())
        article_by_link[rec["link"]] = re.sub(r"\s+", " ", rec.get("text") or "")
    misses = []
    cache_text = {}
    for row in A:
        url = row["source_url"]
        if url not in cache_text:
            if url in article_by_link:
                cache_text[url] = article_by_link[url]
            else:
                cp = cache_path(url)
                if not cp.exists():
                    misses.append((row["trading_name"], "NO CACHE", url))
                    continue
                data = cp.read_bytes()
                if data[:4] == b"%PDF":
                    text = subprocess.check_output(
                        ["pdftotext", "-layout", "-f", "1", "-l", "20", str(cp), "-"],
                        stderr=subprocess.DEVNULL,
                    ).decode("utf-8", "replace")
                else:
                    text = BeautifulSoup(data, "lxml").get_text(" ", strip=True)
                cache_text[url] = re.sub(r"\s+", " ", text)
        blob = cache_text[url]
        ev = row["evidence"]
        if ev not in blob and ev[:80] not in blob:
            misses.append((row["trading_name"], ev[:160], url[-70:]))
    if misses:
        print("EVIDENCE MISSES", len(misses))
        for m in misses:
            print(" -", m[0])
            print("   ", m[1])
        raise SystemExit("evidence not found on source")


def init_db(con):
    con.executescript("""
    DROP TABLE IF EXISTS source_urls;
    DROP TABLE IF EXISTS developers;
    CREATE TABLE developers (
      id INTEGER PRIMARY KEY,
      tier TEXT NOT NULL CHECK (tier IN ('A','B','C')),
      legal_name TEXT,
      trading_name TEXT,
      abn TEXT,
      entity_kind TEXT,
      contact_name TEXT,
      contact_role TEXT,
      public_phone TEXT,
      public_email TEXT,
      website TEXT,
      locality TEXT,
      region_tag TEXT NOT NULL CHECK (region_tag IN ('swis','horizon','off_grid','unknown')),
      scale_note TEXT,
      source_url TEXT NOT NULL,
      source_title TEXT,
      evidence TEXT NOT NULL,
      retrieved_at TEXT NOT NULL,
      duplicate_of INTEGER
    );
    CREATE TABLE source_urls (
      developer_id INTEGER NOT NULL,
      url TEXT NOT NULL,
      retrieved_at TEXT NOT NULL
    );
    """)


def main():
    dap_rows()
    verify_evidence()
    if DB.exists():
        DB.unlink()
    con = sqlite3.connect(DB)
    init_db(con)
    for row in A:
        if row["public_email"] if False else False:
            pass
        cur = con.execute(
            """INSERT INTO developers
            (tier, legal_name, trading_name, abn, entity_kind, contact_name, contact_role,
             public_phone, public_email, website, locality, region_tag, scale_note,
             source_url, source_title, evidence, retrieved_at, duplicate_of)
            VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,NULL)""",
            (row["tier"], row["legal_name"], row["trading_name"], None, row["entity_kind"],
             None, None, None, None, None, row["locality"], row["region_tag"], row["scale_note"],
             row["source_url"], row["source_title"], row["evidence"], RET),
        )
        did = cur.lastrowid
        urls = [row["source_url"]] + row["extra"]
        seen = set()
        for u in urls:
            if not u or u in seen:
                continue
            seen.add(u)
            con.execute("INSERT INTO source_urls (developer_id, url, retrieved_at) VALUES (?,?,?)", (did, u, RET))
    n_c = load_builders(con)
    con.commit()
    # quality checks
    bad_email = con.execute("SELECT COUNT(*) FROM developers WHERE public_email IS NOT NULL").fetchone()[0]
    bad_src = con.execute("SELECT COUNT(*) FROM developers WHERE source_url IS NULL OR retrieved_at IS NULL OR evidence IS NULL OR evidence=''").fetchone()[0]
    if bad_email or bad_src:
        raise SystemExit(f"quality fail email={bad_email} src={bad_src}")
    print("A/B inserted", len(A), "tier C inserted", n_c)
    con.close()

if __name__ == "__main__":
    main()
