Browse Source

update tiles fetching

master
Markus Bergholz 12 months ago
parent
commit
1c654ab759
3 changed files with 72 additions and 12 deletions
  1. +14
    -0
      osm/generate_parts.py
  2. +2
    -12
      osm/getTiles.py
  3. +56
    -0
      osm/getTilesPart.py

+ 14
- 0
osm/generate_parts.py View File

@ -0,0 +1,14 @@
import redis
import json
r = redis.Redis()
def getParts(z):
return z - 7
for z in range(8,11):
parts = getParts(z)
for part in range(0,parts):
obj = {'z': z, 'part': part, 'parts': parts}
print(obj)
r.lpush('parts', json.dumps(obj))

+ 2
- 12
osm/getTiles.py View File

@ -7,7 +7,6 @@ import pngquant
from random import randrange
import redis
r = redis.Redis(host="redis")
server = "88.198.17.138"
HOME = '/mnt/osm/'
@ -28,14 +27,9 @@ def get_pngs(zoom):
progress = progressbar.ProgressBar()
max_x = int(pow(2,zoom))
for x in progress(range(0, max_x)):
if r.get("stop"):
break
r.set("x", x)
for y in range(0, int(pow(2,zoom))):
if r.get("stop"):
break
Path(HOME + 'optimized/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
Path(HOME + 'origin/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
img_data = requests.get(base_url(zoom, x, y), headers=headers).content
png = pngquant.quant_data(data=img_data)
with open(OUTPUT.format(PNG='optimized',Z=zoom,X=x,Y=y), 'wb') as handler:
@ -44,13 +38,9 @@ def get_pngs(zoom):
else:
handler.write(img_data)
r.set("y", y)
if __name__ == '__main__':
for zoom in range(0,9):
if r.get("stop"):
break
r.set('zoom', zoom)
for zoom in range(0,8):
print("fetching " + str(pow(2,zoom) * pow(2,zoom)) + " tiles" )
print("get zoom level: " + str(zoom))
get_pngs(zoom)


+ 56
- 0
osm/getTilesPart.py View File

@ -0,0 +1,56 @@
from math import pow
import requests
import sys
import progressbar
from pathlib import Path
import pngquant
from random import randrange
import redis
import json
r = redis.Redis(host="redis")
server = "88.198.17.138"
HOME = '/mnt/osm/'
OUTPUT = HOME + '{PNG}/{Z}/{X}/{Y}.png'
pngquant.config(min_quality=50, max_quality=75)
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0'
}
def base_url(zoom, x, y):
BASE_URL = 'http://{SERVER}/tile/{Z}/{X}/{Y}.png'
return BASE_URL.format(Z=zoom, X=x, Y=y, SERVER=server)
def get_pngs(zoom, part, parts):
progress = progressbar.ProgressBar()
max_x = int(pow(2,zoom))
for x in progress(range(part, max_x, parts)):
for y in range(0, int(pow(2,zoom))):
Path(HOME + 'optimized/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
img_data = requests.get(base_url(zoom, x, y), headers=headers).content
png = pngquant.quant_data(data=img_data)
with open(OUTPUT.format(PNG='optimized',Z=zoom,X=x,Y=y), 'wb') as handler:
if len(png[1]) < len(img_data):
handler.write(png[1])
else:
handler.write(img_data)
if __name__ == '__main__':
# parts = 3
# part = 0:parts-1
# python3 getTilesPart.py 10 1 2
while r.llen('parts') != 0:
obj = json.loads(r.lpop('parts'))
zoom = obj['z']
part = obj['part']
parts = obj['parts']
print("fetching " + str(pow(2,zoom) * pow(2,zoom)) + " tiles" )
print("get zoom level: " + str(zoom))
get_pngs(zoom, part, parts)

Loading…
Cancel
Save