Who am I?

Freelance geospatial consultant: Spatialised

Organising committee for FOSS4G SotM Oceania 2018 and 2019

OSGeo charter member, marketing committee, OGC representative

Field scientist turned data wrangler

What do I do now with PDAL and Entwine?

Analysis

Processing

PDAL and Entwine in Hydrography

def xyzcoverage(inputfile):
    """
    Provide a file name with extension:
    - xyz
    ...extract a tight polygon around the XY points in the file
    and return a GeoJSON polygon
    """
    pipeline = {
        "pipeline": [
            {
                "type": "readers.text",
                "header": "X\tY\tZ\tFlightllineID\tIntensity",
                "spatialreference": "EPSG:4326",
                "filename": inputfile
            },
            {
                "type": "filters.hexbin",
                "threshold": 1
            }
        ]
    }

def xyzdensity(inputfile):
"""
.xyz files (ascii point clouds or grids)
"""

pointcoverage = getpointcoverage.getpointcoverage(inputfile)
pointcoverage = shape(json.loads(pointcoverage))

# get the area of the coverage. first convert from WGS84 to utm
projstring = geotransforms.guessutm(pointcoverage)

utmcoverage = geotransforms.latlontoutm(pointcoverage, projstring)

covered = utmcoverage.area

npoints = mapcount(inputfile)

meandensity = covered/npoints

return json.dumps({
    'meandensity': meandensity,
    'area': covered,
    'npoints': npoints
})

https://github/frontiersi/qa4mbes-data-pipeline/blob/master/notebooks/

{
  "pipeline":[
    {
        "type" : "readers.text",
        "filename" : "inputfile.txt",
        "separator" : ",",
        "header" : "X,Y,Z",
        "skip": 1
    },
    {
        "type" : "filters.reprojection",
        "in_srs":"EPSG:4326",
        "out_srs":"EPSG:3577"
    },
    {
        "type": "filters.ferry",
        "dimensions": "=>Classification, =>PointSourceId"
    },
    {
        "type": "filters.assign",
        "assignment": "PointSourceId[:]=1, Classification[:]=2"
    },
    {
        "type": "filters.outlier"
    },
    {
        "type":"filters.transformation",
        "matrix": "1 0 0 0 0 1 0 0 0 0 -1 0 0 0 0 1"
    },
    {
        "type":"filters.colorinterp",
        "minimum" : -8100,
        "maximum" : 0,
        "ramp" : "blue_hue",
        "dimension":"Z"
    },
    {
      "type":"writers.las",
      "filename":"outfile.laz",
      "minor_version": 4,
      "a_srs": "EPSG:3577"
    }
  ]
}

#!/bin/bash

path=$1
table=$2

echo $table

for filename in $path/*.laz; do

    echo $filename
    thefile=$(basename $filename .laz)
    echo $thefile
    bounds=$(pdal info --boundary $filename | jq '.boundary.boundary')
    bounds=$(echo $bounds | tr -d \")
    pointcount=$(pdal info --metadata $filename | jq '.metadata.count' )
    echo $pointcount
    thedepth=$(pdal info --stats $filename | jq '.stats.statistic[2].average' )
    echo $thedepth
    #allthejson=$(pdal info --summary $filename )

    psql -U adam -h localhost -p 5432 -d bathymetryindex -c \
    "insert into $table (filename, boundingbox, npoints, meandepth )
    values ('$thefile', ST_GeomFromText('$bounds'), $pointcount, $thedepth);"

done

Workflow

Read ASCII from a THREDDS server

Reproject, invert Z, LAZ-ify

Collect boundaries and postGIS-ify

Entwine-ify

Do stuff...

{
"pipeline": [
{
   "type": "readers.ept",
   "filename": "https://act-2015-rgb.s3.amazonaws.com/",
   "bounds": “([-2639580, -2635580], [-4137200,-4142200])"
},
{
   "type":"writers.gdal",
   "gdalopts":"t_srs=EPSG:3577",
   “resolution":20,
   “filename”:"diamantina-dsm.tiff",
   "output_type":"idw"
}
]
}

Making landscape-scales easy

{
  "pipeline": [
    {
        "type": "readers.ept",
        "filename": "http://act-2015-rgb.s3.amazonaws.com/“,
        "bounds": "([692738, 692967], [6092255,6092562])"
    },
    {
        “type”:”filters.hag"
    },
    {
        "type": "filters.ferry",
        "dimensions": "HeightAboveGround=>Z"
    },
    {
        “type”:”filters.crop”,
        "polygon" : "POLYGON((692809.631984224 6092391.60313353,...,692809.631984224 6092391.60313353))"
    },
    "cropped-data-from-entwine.laz"
  ]
}

Workflow

Colourize

Entwine-ify

Do stuff...

{
"pipeline": [
{
  "type":"readers.ept",
  "filename":"https://lower-mdb-2013-rgb.s3.amazonaws.com/",
  "bounds":"([566655, 567655], [6254307,6255307])"
},
{
    "type":"filters.python",
    "script":"treepercentage.py",
    "function":"treepercentage",
    "module":"anything",
    "pdalargs":"{\"cellsize\":10,
                 \"spatialreference\":\"EPSG:28354\",
                 \"outputfile\":\"treeness-new.tiff\"}"
},
{
    "type":"filters.range",
    "limits":"Classification[2:2]"
},
{
    "type":"writers.gdal",
    "gdalopts":"t_srs=EPSG:28355",
    "resolution":10,
    "filename":"trees-dem.tiff",
    "output_type":"idw"
}
]
}

Workflow

Colourize

Entwine-ify

Do stuff!

Scale mixing and tiny airplanes

Workflow

Find coregistration points, generate transformation matrix, apply trasformation matrix

Entwine-ify

Do stuff...

Workflow

Classify ground in RPA data

Generate transform with ICP

Apply transform

Entwine-ify

Do stuff...

Open by default