Browse Source

prepare tiles saving

master
Markus Bergholz 1 year ago
parent
commit
2354761c4c
2 changed files with 112 additions and 12 deletions
  1. +22
    -12
      osm/getTiles.py
  2. +90
    -0
      osm/getTilesThreaded.py

+ 22
- 12
osm/getTiles.py View File

@ -5,10 +5,12 @@ import progressbar
from pathlib import Path
import pngquant
from random import randrange
import redis
server = ['a','b','c']
r = redis.Redis()
server = "88.198.17.138"
FOLDER = '/home/m/osm'
HOME = '/mnt/osm'
OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
pngquant.config(min_quality=50, max_quality=80)
@ -19,24 +21,32 @@ headers = {
def base_url(zoom, x, y):
BASE_URL = 'https://{SERVER}.tile.openstreetmap.org/{Z}/{X}/{Y}.png'
BASE_URL = 'http://{SERVER}/tile/{Z}/{X}/{Y}.png'
return BASE_URL.format(Z=zoom, X=x, Y=y, SERVER=server[randrange(3)])
def get_pngs(zoom):
progress = progressbar.ProgressBar()
max_x = int(pow(2,zoom)-1)
for x in range(0, max_x):
print(str(x) + '/' + str(max_x))
progress = progressbar.ProgressBar()
for y in progress(range(0, int(pow(2,zoom)-1))):
Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
for x in progress(range(0, max_x)):
if r.get("stop"):
break
r.set("x", x)
for y in range(0, int(pow(2,zoom)-1)):
if r.get("stop"):
break
Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
img_data = requests.get(base_url(zoom, x, y), headers=headers).content
png = pngquant.quant_data(data=img_data)
with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
handler.write(png[1])
r.set("y", y)
if __name__ == '__main__':
zoom = int(sys.argv[1])
print("fetching " + str(pow(2,zoom)+ pow(2,zoom)) + " tiles" )
get_pngs(zoom)
for zoom in range(0,10):
if r.get("stop"):
break
r.set('zoom', zoom)
print("fetching " + str(pow(2,zoom)+ pow(2,zoom)) + " tiles" )
print("get zoom level: " + str(zoom))
get_pngs(zoom)

+ 90
- 0
osm/getTilesThreaded.py View File

@ -0,0 +1,90 @@
from math import pow
import requests
import sys
import progressbar
from pathlib import Path
import pngquant
from random import randrange
import threading
import time
import q
import os
import random
import string
def rnd_file():
# printing lowercase
letters = string.ascii_lowercase
return "/tmp/" + ( ''.join(random.choice(letters) for i in range(32)) ) + ".png"
DEFAULT_WORKER = 5
thread = []
server = "88.198.17.138"
HOME = '/mnt/osm'
OUTPUT = HOME + '/{Z}/{X}/{Y}.png'
pngquant.config(min_quality=50, max_quality=80)
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Fedora; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0'
}
def base_url(zoom, x, y):
BASE_URL = 'http://{SERVER}/tile/{Z}/{X}/{Y}.png'
return BASE_URL.format(Z=zoom, X=x, Y=y, SERVER=server[randrange(3)])
def get_and_process_tile(x=None,zoom=None):
if None not in [x, zoom]:
pngquant.config(min_quality=50, max_quality=80, tmp_file=rnd_file())
q("using " + pngquant.config()["tmp_file"])
for y in range(0, int(pow(2,zoom)-1)):
q(str(x) + ',' + str(y) + ',' + str(zoom))
# Path(HOME + '/{Z}/{X}'.format(X=x,Z=zoom)).mkdir(parents=True, exist_ok=True)
# img_data = requests.get(base_url(zoom, x, y), headers=headers).content
# png = pngquant.quant_data(data=img_data)
# with open(OUTPUT.format(Z=zoom,X=x,Y=y), 'wb') as handler:
# handler.write(png[1])
try:
os.remove(pngquant.config()["tmp_file"])
except:
return
return
def get_pngs(zoom):
max_x = int(pow(2,zoom)-1)
progress = progressbar.ProgressBar()
for x in progress(range(0, max_x)):
n = -1
while True:
n = n + 1
# check if a worker is done and update status
if thread[n % DEFAULT_WORKER].isAlive() is False:
q("worker " + str(n % DEFAULT_WORKER) + " is free now")
break
else:
q("no free worker available ...")
time.sleep(2.0)
thread[n % DEFAULT_WORKER] = threading.Thread(target=get_and_process_tile, args=[x, zoom])
thread[n % DEFAULT_WORKER].start()
q("worker " + str(n % DEFAULT_WORKER) + " busy with " + str(x))
if __name__ == '__main__':
zoom = int(sys.argv[1])
# init 5 threads
for n in range(0, DEFAULT_WORKER):
thread.append(threading.Thread(target=get_and_process_tile, args=[None,None]))
thread[n].start()
print("spawning thread " + str(n+1) + "/" + str(DEFAULT_WORKER))
print("fetching " + str(pow(2,zoom)+ pow(2,zoom)) + " tiles" )
get_pngs(zoom)

Loading…
Cancel
Save