From c577e996bb4c3aa7881ea1fe0a0a8397ed33553f Mon Sep 17 00:00:00 2001 From: Robert Coup Date: Wed, 5 Sep 2018 10:01:04 +0100 Subject: [PATCH] Updates to Dockerfile: * Docker image now runs a working validator service. * Enable customisable listen interface & size limit (via environment variables) * Update to GDAL 2.3.1 * Enable ZSTD compression support * Redirect `/` to `/html` `docker run --rm -p 5000:5000 cog_validator` will now run the service on http://localhost:5000/ --- Dockerfile | 90 ++++++++++++++++++++++++--------------------- README.md | 10 +++++ cog_validator.py | 21 +++++++---- templates/main.html | 4 +- 4 files changed, 75 insertions(+), 50 deletions(-) diff --git a/Dockerfile b/Dockerfile index 128a426..2965bd8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,33 +12,33 @@ RUN \ automake16 \ libcurl-devel -# Fetch PROJ.4 - RUN \ - curl -L http://download.osgeo.org/proj/proj-4.9.3.tar.gz | tar zxf - -C /tmp + # Build zstd + curl -L https://github.com/facebook/zstd/archive/v1.3.3.tar.gz | tar zxf - -C /tmp \ + && cd /tmp/zstd-1.3.3/lib \ + && make -j3 PREFIX=/var/task ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 \ + && make install PREFIX=/var/task ZSTD_LEGACY_SUPPORT=0 CFLAGS=-O1 \ + && cd /tmp \ + && rm -rf /tmp/zstd-1.3.3 -# Build and install PROJ.4 - -WORKDIR /tmp/proj-4.9.3 +# Fetch PROJ.4 RUN \ - ./configure \ - --prefix=/var/task && \ - make -j $(nproc) && \ - make install + curl -L http://download.osgeo.org/proj/proj-4.9.3.tar.gz | tar zxf - -C /tmp \ + && cd /tmp/proj-4.9.3 \ + && ./configure --prefix=/var/task \ + && make -j $(nproc) \ + && make install \ + && cd /tmp \ + && rm -rf /tmp/proj-4.9.3 # Fetch GDAL RUN \ - mkdir -p /tmp/gdal && \ - curl -L http://download.osgeo.org/gdal/2.2.1/gdal-2.2.1.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 - -# Build + install GDAL - -WORKDIR /tmp/gdal - -RUN \ - ./configure \ + mkdir -p /tmp/gdal \ + && curl -L http://download.osgeo.org/gdal/2.3.1/gdal-2.3.1.tar.gz | tar zxf - -C /tmp/gdal --strip-components=1 \ + && cd /tmp/gdal \ + && ./configure \ --prefix=/var/task \ --datarootdir=/var/task/share/gdal \ --with-jpeg=internal \ @@ -48,12 +48,13 @@ RUN \ --without-pcraster \ --without-png \ --without-gif \ - --without-pcidsk && \ - make -j $(nproc) && \ - cd swig/python && \ - make && \ - cd ../.. && \ - make install + --with-zstd=/var/task \ + --without-pcidsk \ + && make -j $(nproc) \ + && cd swig/python \ + && make \ + && cd ../.. \ + && make install # Install Python deps in a virtualenv @@ -68,30 +69,37 @@ COPY requirements.txt /var/task/requirements.txt RUN pip install -r requirements.txt +# Lambda stuff # Add GDAL libs to the function zip - RUN \ - strip lib/libgdal.so.20.3.0 && \ - strip lib/libproj.so.12.0.0 + strip lib/libgdal.so.20.4.1 && \ + strip lib/libproj.so.12.0.0 && \ + strip lib/libzstd.so.1.3.3 RUN \ zip --symlinks \ -r /tmp/task.zip \ lib/libgdal.so* \ lib/libproj.so* \ + lib/libzstd.so* \ share/gdal/ - # Add Python deps to the function zip - -WORKDIR /tmp/virtualenv/lib/python2.7/site-packages - -RUN find . -name \*.so\* -exec strip {} \; - RUN \ - zip -r /tmp/task.zip flask werkzeug jinja2 markupsafe itsdangerous.py* click requests idna chardet certifi urllib3 osgeo - -WORKDIR /tmp/gdal/swig/python/build/lib.linux-x86_64-2.7 - -RUN find . -name \*.so\* -exec strip {} \; - -RUN zip -r /tmp/task.zip . + cd /tmp/virtualenv/lib/python2.7/site-packages \ + && find . -name \*.so\* -exec strip {} \; \ + && zip -r /tmp/task.zip flask werkzeug jinja2 markupsafe itsdangerous.py* click requests idna chardet certifi urllib3 osgeo \ + && cd /tmp/gdal/swig/python/build/lib.linux-x86_64-2.7 \ + && find . -name \*.so\* -exec strip {} \; \ + && zip -r /tmp/task.zip . + +# Local execution +RUN pip install /tmp/gdal/swig/python + +WORKDIR /tmp/virtualenv +COPY templates/ /tmp/virtualenv/templates/ +COPY cog_validator.py /tmp/virtualenv/ +COPY validate_cloud_optimized_geotiff.py /tmp/virtualenv/ +ENV COG_LIMIT=50 +ENV LISTEN=0.0.0.0 +EXPOSE 5000 +CMD ["/tmp/virtualenv/bin/python", "/tmp/virtualenv/cog_validator.py"] diff --git a/README.md b/README.md index 93df56d..87d248f 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,16 @@ For all the above interfaces, the query will return a JSon document with the fol The service expose a basic HTML page for users to submit their GeoTIFF files and display the result of the validation +## Local + +``` +$ docker build -t cog_validator . +$ docker run --rm -p 5000:5000 cog_validator +``` +Then access at http://localhost:5000/ + +The upload image size limit defaults to 50MB; you can increase it via `-e COG_LIMIT={MB}` + ## AWS Lambda / API Gateway The service can be deployed as a AWS Lamba function, accessible through the AWS API Gateway. diff --git a/cog_validator.py b/cog_validator.py index c5ac116..97b2325 100755 --- a/cog_validator.py +++ b/cog_validator.py @@ -24,15 +24,16 @@ import json import os -from flask import Flask, request as flask_request, render_template +from flask import Flask, request as flask_request, render_template, redirect from werkzeug.exceptions import RequestEntityTooLarge import requests from osgeo import gdal import validate_cloud_optimized_geotiff - + app = Flask(__name__) # http://docs.aws.amazon.com/lambda/latest/dg/limits.html -app.config['MAX_CONTENT_LENGTH'] = 6 * 1024 * 1024 +COG_LIMIT = int(os.environ.get('COG_LIMIT', '6')) +app.config['MAX_CONTENT_LENGTH'] = COG_LIMIT * 1024 * 1024 tmpfilename = '/tmp/cog_validator_tmp.tif' @@ -101,8 +102,8 @@ def validate(args): return json.dumps({'status': 'success', 'gdal_info' : info, 'details': details}), 200, { "Content-Type": "application/json" } else: return json.dumps({'status': 'failure', 'gdal_info' : info, 'details': details, 'validation_errors': errors}), 400, { "Content-Type": "application/json" } - - + + @app.route('/api/validate', methods=['GET', 'POST']) def api_validate(): if flask_request.method == 'POST': @@ -172,7 +173,7 @@ def html(): root_url = flask_request.url_root[0:-1] if 'AWS_API_GATEWAY_STAGE' in flask_request.environ: root_url += '/' + flask_request.environ['AWS_API_GATEWAY_STAGE'] - return render_template('main.html', root_url = root_url) + return render_template('main.html', root_url = root_url, limit=COG_LIMIT) @app.route('/html/validate', methods=['POST']) def html_validate(): @@ -200,9 +201,15 @@ def html_validate(): errors = ret['validation_errors'] return render_template('result.html', root_url = root_url, global_result = global_result, errors = errors) +@app.route('/', methods=['GET']) +def root(): + return redirect('/html') + + # We only need this for local development. env = os.environ DEBUG = env.get('DEBUG', 'False') +LISTEN = env.get('LISTEN', '127.0.0.1') if __name__ == '__main__': - app.run(debug=DEBUG=="True") + app.run(debug=DEBUG=="True", host=LISTEN) diff --git a/templates/main.html b/templates/main.html index 0dbb416..74c2291 100644 --- a/templates/main.html +++ b/templates/main.html @@ -57,8 +57,8 @@

Cloud optimized GeoTIFF validator

var reader = new FileReader(); reader.onloadend = function (evt) { b64_length = evt.target.result.length - if( b64_length > 6 * 1024 * 1024) { - alert('File is too big. Max 6 MB allowed') + if( b64_length > {{ limit }} * 1024 * 1024) { + alert('File is too big. Max {{ limit }} MB allowed') return } document.getElementById("file_b64").value = evt.target.result;