Browse Source

Fix collation errors

master
Oliver Schrenk 8 years ago
parent
commit
1720275003
  1. 155
      README.md
  2. 24
      mysql-apidb06.sql

155
README.md

@ -188,4 +188,157 @@ and got
You have to import a valid schema before you can poulate the database. Unfortunately MySQL is no longer officialy supported, so there is no official schema available.
See above for my attempts to create one.
See above for my attempts to create one.
### Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry 'krautundrueber' for key 'users_display_name_idx' ###
After I changed some name of columns the imort seemed to work fine.
Executing
osmosis --read-xml file="bremen.osm.bz2" --write-apidb-0.6 host="127.0.0.1" dbType="mysql" database="api06_test" user="osm" password="osm" validateSchemaVersion=no
works for a while but fails with
...
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry 'krautundrueber' for key 'users_display_name_idx'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.Util.getInstance(Util.java:386)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1039)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3609)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:3541)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2002)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2163)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2624)
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2127)
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2427)
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2345)
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2330)
at org.openstreetmap.osmosis.apidb.v0_6.impl.UserManager.insertUser(UserManager.java:140)
... 18 more
On first look on entries look fine. So I had to dive deeper into Osmosis.
I build Osmosis and imported it into Eclipse. Unfortunately the current project layout doesn't allow for launching any task. I had to create a Junit-Test in th `apidb` module, to have access to all plugins.
@Test
public void debugDuplicateEntry() {
Osmosis.run(new String[] {
"--read-xml-0.6",
"/Users/q2web/Downloads/bremen.osm",
"--write-apidb-0.6",
"host=127.0.0.1",
"dbType=mysql",
"database=api06_test",
"user=osm",
"password=osm",
"validateSchemaVersion=no"
});
}
The user that makes problem is
uid="341865" user="krautundrueber"
It clashes with
uid="14181" user="Krautundrueber"
While the user user has a unique id, it seems that the index `users_display_name_idx` on `varchar(255)`, ignores case.
So one has to set case sensitive collation
...
WORD VARCHAR(128) CHARACTER SET utf8 COLLATE utf8_cs
...
Unfortunately the Collator `utf8_cs` isn't installed per default, so I tried `utf8_bin`
Setting the default collate didn't help
CREATE DATABASE IF NOT EXISTS api06_test
DEFAULT CHARACTER SET utf8
DEFAULT COLLATE utf8_bin;
so I also set it expclitly for the `display_name` column
`display_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
### com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry '54750854-4-FIXME' for key 'PRIMARY' ###
After fixing the character set on the user names, the same error happens on the tags for the ways
Caused by: com.mysql.jdbc.exceptions.jdbc4.MySQLIntegrityConstraintViolationException: Duplicate entry '54750854-4-FIXME' for key 'PRIMARY'
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:57)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:525)
at com.mysql.jdbc.Util.handleNewInstance(Util.java:411)
at com.mysql.jdbc.Util.getInstance(Util.java:386)
at com.mysql.jdbc.SQLError.createSQLException(SQLError.java:1040)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4074)
at com.mysql.jdbc.MysqlIO.checkErrorPacket(MysqlIO.java:4006)
at com.mysql.jdbc.MysqlIO.sendCommand(MysqlIO.java:2468)
at com.mysql.jdbc.MysqlIO.sqlQueryDirect(MysqlIO.java:2629)
at com.mysql.jdbc.ConnectionImpl.execSQL(ConnectionImpl.java:2719)
at com.mysql.jdbc.PreparedStatement.executeInternal(PreparedStatement.java:2155)
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2450)
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2371)
at com.mysql.jdbc.PreparedStatement.executeUpdate(PreparedStatement.java:2355)
at org.openstreetmap.osmosis.apidb.v0_6.ApidbWriter.flushWayTags(ApidbWriter.java:724)
... 20 more
This is because some tags are called `FIXME` and some are `fixme`.
So set the collation for the keys in `way_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
...
While were at it do the same for `node_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
...
and `relation_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
...
and `current_relation_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
...
and `changeset_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
...
and `current_node_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
...
and `current_way_tags`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
...
and `user_preferences`
...
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
...

24
mysql-apidb06.sql

@ -18,7 +18,9 @@
-- Create schema api06_test
--
CREATE DATABASE IF NOT EXISTS api06_test;
CREATE DATABASE IF NOT EXISTS api06_test
DEFAULT CHARACTER SET utf8
DEFAULT COLLATE utf8_bin;
USE api06_test;
--
@ -30,7 +32,7 @@ CREATE TABLE `api06_test`.`acls` (
`id` int(11) NOT NULL auto_increment,
`address` int(10) unsigned NOT NULL,
`netmask` int(10) unsigned NOT NULL,
`k` varchar(255) NOT NULL,
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
`v` varchar(255) default NULL,
PRIMARY KEY (`id`),
KEY `acls_k_idx` (`k`)
@ -53,7 +55,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`changeset_tags`;
CREATE TABLE `api06_test`.`changeset_tags` (
`changeset_id` bigint(64) NOT NULL,
`k` varchar(255) NOT NULL default '',
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`v` varchar(255) NOT NULL default '',
KEY `changeset_tags_id_idx` (`changeset_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@ -103,7 +105,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`current_node_tags`;
CREATE TABLE `api06_test`.`current_node_tags` (
`node_id` bigint(64) NOT NULL,
`k` varchar(255) NOT NULL default '',
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`v` varchar(255) NOT NULL default '',
PRIMARY KEY (`node_id`,`k`),
CONSTRAINT `current_node_tags_ibfk_1` FOREIGN KEY (`node_id`) REFERENCES `current_nodes` (`id`)
@ -183,7 +185,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`current_relation_tags`;
CREATE TABLE `api06_test`.`current_relation_tags` (
`id` bigint(64) NOT NULL,
`k` varchar(255) NOT NULL default '',
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`v` varchar(255) NOT NULL default '',
PRIMARY KEY (`id`,`k`),
CONSTRAINT `current_relation_tags_ibfk_1` FOREIGN KEY (`id`) REFERENCES `current_relations` (`id`)
@ -258,7 +260,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`current_way_tags`;
CREATE TABLE `api06_test`.`current_way_tags` (
`way_id` bigint(64) NOT NULL,
`k` varchar(255) NOT NULL default '',
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`v` varchar(255) NOT NULL default '',
PRIMARY KEY (`way_id`,`k`),
CONSTRAINT `current_way_tags_ibfk_1` FOREIGN KEY (`way_id`) REFERENCES `current_ways` (`id`)
@ -498,7 +500,7 @@ DROP TABLE IF EXISTS `api06_test`.`node_tags`;
CREATE TABLE `api06_test`.`node_tags` (
`node_id` bigint(64) NOT NULL,
`version` bigint(20) NOT NULL,
`k` varchar(255) NOT NULL default '',
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`v` varchar(255) NOT NULL default '',
PRIMARY KEY (`node_id`,`version`,`k`),
CONSTRAINT `node_tags_ibfk_1` FOREIGN KEY (`node_id`, `version`) REFERENCES `nodes` (`node_id`, `version`)
@ -579,7 +581,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`relation_tags`;
CREATE TABLE `api06_test`.`relation_tags` (
`relation_id` bigint(64) NOT NULL default '0',
`k` varchar(255) NOT NULL default '',
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`v` varchar(255) NOT NULL default '',
`version` bigint(20) NOT NULL,
PRIMARY KEY (`relation_id`,`version`,`k`),
@ -700,7 +702,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`user_preferences`;
CREATE TABLE `api06_test`.`user_preferences` (
`user_id` bigint(20) NOT NULL,
`k` varchar(255) NOT NULL,
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
`v` varchar(255) NOT NULL,
PRIMARY KEY (`user_id`,`k`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
@ -751,7 +753,7 @@ CREATE TABLE `api06_test`.`users` (
`active` int(11) NOT NULL default '0',
`pass_crypt` varchar(255) NOT NULL,
`creation_time` datetime NOT NULL,
`display_name` varchar(255) NOT NULL default '',
`display_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL default '',
`data_public` tinyint(1) NOT NULL default '0',
`description` text NOT NULL,
`home_lat` double default NULL,
@ -812,7 +814,7 @@ UNLOCK TABLES;
DROP TABLE IF EXISTS `api06_test`.`way_tags`;
CREATE TABLE `api06_test`.`way_tags` (
`way_id` bigint(64) NOT NULL default '0',
`k` varchar(255) NOT NULL,
`k` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
`v` varchar(255) NOT NULL,
`version` bigint(20) NOT NULL,
PRIMARY KEY (`way_id`,`version`,`k`),

Loading…
Cancel
Save