Browse Source

update osm

master
Markus Bergholz 12 months ago
parent
commit
d16be57871
3 changed files with 54 additions and 19 deletions
  1. +7
    -0
      osm/Dockerfile
  2. +12
    -7
      osm/getTiles.py
  3. +35
    -12
      osm/getTilesThreaded.py

+ 7
- 0
osm/Dockerfile View File

@ -0,0 +1,7 @@
FROM python:3-slim
RUN apt update && apt install pngquant -y
RUN pip install redis pngquant progressbar requests q
COPY get* /

+ 12
- 7
osm/getTiles.py View File

@ -7,12 +7,12 @@ import pngquant
from random import randrange
import redis
r = redis.Redis()
r = redis.Redis(host="redis")
server = "88.198.17.138"
HOME = '/mnt/osm'
HOME = '/mnt/osm/'
OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
OUTPUT = HOME + '{PNG}/{Z}/{X}/{Y}.png'
pngquant.config(min_quality=50, max_quality=80)
headers = {
@ -34,15 +34,20 @@ def get_pngs(zoom):
for y in range(0, int(pow(2,zoom))):
if r.get("stop"):
break
Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
Path(HOME + 'optimized/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
Path(HOME + 'origin/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
img_data = requests.get(base_url(zoom, x, y), headers=headers).content
png = pngquant.quant_data(data=img_data)
with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
handler.write(png[1])
with open(OUTPUT.format(PNG='optimized',Z=zoom,X=x,Y=y), 'wb') as handler:
if len(png[1]) < len(img_data):
handler.write(png[1])
else:
handler.write(img_data)
r.set("y", y)
if __name__ == '__main__':
for zoom in range(0,10):
for zoom in range(0,9):
if r.get("stop"):
break
r.set('zoom', zoom)


+ 35
- 12
osm/getTilesThreaded.py View File

@ -3,7 +3,6 @@ import requests
import sys
import progressbar
from pathlib import Path
import pngquant
from random import randrange
import threading
import time
@ -21,10 +20,10 @@ thread = []
server = "88.198.17.138"
HOME = '/mnt/osm'
HOME = '/mnt/osm/optimized'
OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
pngquant.config(min_quality=50, max_quality=80)
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0'
@ -36,20 +35,44 @@ def base_url(zoom, x, y):
def get_and_process_tile(x=None,zoom=None):
if None not in [x, zoom]:
pngquant.config(min_quality=50, max_quality=80, tmp_file=rnd_file())
q("using " + pngquant.config()["tmp_file"])
import pngquant
N = 0
while True:
try:
pngquant.config(min_quality=50, max_quality=80, tmp_file=rnd_file())
break
except:
N += 1
if N > 10:
break
continue
for y in range(0, int(pow(2,zoom))):
q(str(x) + ',' + str(y) + ',' + str(zoom))
# Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
# img_data = requests.get(base_url(zoom, x, y), headers=headers).content
# png = pngquant.quant_data(data=img_data)
# with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
# handler.write(png[1])
data = requests.get(base_url(zoom, x, y), headers=headers).content
N = 0
while True:
try:
png = pngquant.quant_data(data=data)
break
except:
N += 1
if N > 10:
break
continue
with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
if len(png[1]) < len(data):
handler.write(png[1])
else:
handler.write(data)
try:
os.remove(pngquant.config()["tmp_file"])
q("remove successfull")
except:
return
q("remove not possible: " + str(pngquant.config()["tmp_file"]))
return
@ -61,7 +84,7 @@ def get_pngs(zoom):
while True:
n = n + 1
# check if a worker is done and update status
if thread[n % DEFAULT_WORKER].isAlive() is False:
if thread[n % DEFAULT_WORKER].is_alive() is False:
q("worker " + str(n % DEFAULT_WORKER) + " is free now")
break
else:


Loading…
Cancel
Save