a:4:{s:5:"child";a:1:{s:0:"";a:1:{s:3:"rss";a:1:{i:0;a:6:{s:4:"data";s:3:"
";s:7:"attribs";a:1:{s:0:"";a:1:{s:7:"version";s:3:"2.0";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:1:{s:0:"";a:1:{s:7:"channel";a:1:{i:0;a:6:{s:4:"data";s:217:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:1:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:12:"Planet MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:27:"http://www.planetmysql.org/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Fri, 27 Aug 2010 04:45:01 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"language";a:1:{i:0;a:5:{s:4:"data";s:2:"en";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:42:"Planet MySQL - http://www.planetmysql.org/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"item";a:50:{i:0;a:6:{s:4:"data";s:38:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:5:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:80:"テストケースの実行にあわせて Apache を起動・終了する方法";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:69:"http://developer.cybozu.co.jp/kazuho/2010/08/apache-f729.html?lang=en";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:69:"http://developer.cybozu.co.jp/kazuho/2010/08/apache-f729.html?lang=en";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:3191:"ウェブアプリケーションやライブラリの結合テストを行う段階になると、実際に Apache を起動してテストを実行したくなります。しかし、そのためにいちいち Apache の設定ファイルを修正して httpd を再起動して、とやっていては面倒です。特に複数のプログラムを同時に開発していると、あっちをテストしたらこっちが動かなくなって… なんてなったりして嫌気がさしてきます。
そこで、テストを実行する際に、環境毎に異なる以下のような問題を吸収しつつ、テスト専用に設定された Apache を自動的に起動終了してくれる Perl モジュール:Test::Httpd::Apache2 を書きました。
環境によって、インストールパスが違う (/usr/local/apache/bin だったり /usr/sbin だったり)
環境によって LoadModule の要不要や、ロードするパスが違う
環境によってプログラム名が違う (Debian 系では httpd ではなく apache2)
Apache/2.0 と 2.2 でモジュール名が変わっているケースがある (mod_access と mod_authz_host)
使い方は簡単。テストコードの先頭で、Test::Httpd::Apache2 のインスタンスを生成すると、自動的に TCP の空きポートを探して Apache が起動するので、そのアドレスに HTTP クライアントからアクセスするだけです。起動された Apache は、Test::Httpd::Apache2 のインスタンスが解放されるタイミングで終了され、一時ファイルも自動的に消去されます。
use Test::Httpd::Apache2;my $httpd = Test::Httpd::Apache2->new( required_modules => [ qw(perl) ], # mod_perl をロード custom_conf => << 'EOT',<Location /> SetHandler perl-script ...</Location>EOT);my $root_url = "http://" . $httpd->listen . "/"; # 起動した host:port から URL 生成# あとはテスト...
そして、テスト本体が Perl のコードである必要はないので、PHP や他の言語のテストを実行するためのラッパーとして使うこともできます。
use Test::Httpd::Apache2;my $httpd = Test::Httpd::Apache2->new( required_modules => [ qw(php5) ], # mod_php をロード custom_conf => << 'EOT',DocumentRoot "docroot"SetHandler php5-scriptEOT); # 起動した host:port から URL を生成して、環境変数に登録$ENV{ROOT_URL} = "http://" . $httpd->listen . "/";# my_test.php を実行system('php', 'my_test.php') == 0 or die "my_test.php failed: $?";
Test::mysqld や Test::postgresql と組み合わせると、データベースを使うウェブアプリの結合テストが気軽に書けそうで夢が広がりまくりんぐ。
まだ荒削りなモジュールなので、改善提案やパッチ等お待ちしております。インストールは cpan -i Test::Httpd::Apache2、リポジトリは github.com/kazuho/p5-test-httpd-apache2 にあります。
それでは、have fun!";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Fri, 27 Aug 2010 04:10:49 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3976:"
ウェブアプリケーションやライブラリの結合テストを行う段階になると、実際に Apache を起動してテストを実行したくなります。しかし、そのためにいちいち Apache の設定ファイルを修正して httpd を再起動して、とやっていては面倒です。特に複数のプログラムを同時に開発していると、あっちをテストしたらこっちが動かなくなって… なんてなったりして嫌気がさしてきます。
そこで、テストを実行する際に、環境毎に異なる以下のような問題を吸収しつつ、テスト専用に設定された Apache を自動的に起動終了してくれる Perl モジュール:Test::Httpd::Apache2 を書きました。
- 環境によって、インストールパスが違う (/usr/local/apache/bin だったり /usr/sbin だったり)
- 環境によって LoadModule の要不要や、ロードするパスが違う
- 環境によってプログラム名が違う (Debian 系では httpd ではなく apache2)
- Apache/2.0 と 2.2 でモジュール名が変わっているケースがある (mod_access と mod_authz_host)
使い方は簡単。テストコードの先頭で、Test::Httpd::Apache2 のインスタンスを生成すると、自動的に TCP の空きポートを探して Apache が起動するので、そのアドレスに HTTP クライアントからアクセスするだけです。起動された Apache は、Test::Httpd::Apache2 のインスタンスが解放されるタイミングで終了され、一時ファイルも自動的に消去されます。
use Test::Httpd::Apache2;
my $httpd = Test::Httpd::Apache2->new(
required_modules => [ qw(perl) ], # mod_perl をロード
custom_conf => << 'EOT',
<Location />
SetHandler perl-script
...
</Location>
EOT
);
my $root_url = "http://" . $httpd->listen . "/"; # 起動した host:port から URL 生成
# あとはテスト
...
そして、テスト本体が Perl のコードである必要はないので、PHP や他の言語のテストを実行するためのラッパーとして使うこともできます。
use Test::Httpd::Apache2;
my $httpd = Test::Httpd::Apache2->new(
required_modules => [ qw(php5) ], # mod_php をロード
custom_conf => << 'EOT',
DocumentRoot "docroot"
SetHandler php5-script
EOT
);
# 起動した host:port から URL を生成して、環境変数に登録
$ENV{ROOT_URL} = "http://" . $httpd->listen . "/";
# my_test.php を実行
system('php', 'my_test.php') == 0
or die "my_test.php failed: $?";
Test::mysqld や Test::postgresql と組み合わせると、データベースを使うウェブアプリの結合テストが気軽に書けそうで夢が広がりまくりんぐ。
まだ荒削りなモジュールなので、改善提案やパッチ等お待ちしております。インストールは cpan -i Test::Httpd::Apache2、リポジトリは github.com/kazuho/p5-test-httpd-apache2 にあります。
それでは、have fun!
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:10:"Kazuho Oku";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:1;a:6:{s:4:"data";s:78:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:31:"Open Query on Twitter/Identi.ca";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:33:"http://openquery.com/blog/?p=1377";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:52:"http://openquery.com/blog/open-query-twitteridentica";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:381:"Open Query now has its own @openquery account on Twitter and Identi.ca so you can conveniently follow us there for announcements and tips – and also ask us questions! All OQ engineers can post/reply. The OQ site front page also tracks this feed.
Previously I was posting from my personal @arjenlentz account with #openquery hashtag, but that’s obviously less practical.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Fri, 27 Aug 2010 03:47:27 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:8:{i:0;a:5:{s:4:"data";s:13:"Uncategorized";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:4:"feed";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:6:"follow";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:8:"identica";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:7:"mariadb";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:10:"Open Query";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:7:"twitter";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:835:"Open Query now has its own @openquery account on Twitter and Identi.ca so you can conveniently follow us there for announcements and tips – and also ask us questions! All OQ engineers can post/reply. The OQ site front page also tracks this feed.
Previously I was posting from my personal @arjenlentz account with #openquery hashtag, but that’s obviously less practical.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:10:"Open Query";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:2;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:40:"U.S. Linux Cluster Stack training survey";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:35:"https://fghaas.wordpress.com/?p=581";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:79:"http://fghaas.wordpress.com/2010/08/26/u-s-linux-cluster-stack-training-survey/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:378:"After our hugely successful summer class in Silicon Valley, we have set up a survey to help us determine the location of our Q4 DRBD Total course in the United States. So let us know where you would like to see the next incarnation of the industry’s premier Linux High Availability training!
Our survey just takes a minute to complete. We appreciate your input!
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 18:26:30 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:8:"Training";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2292:"After our hugely successful summer class in Silicon Valley, we have set up a survey to help us determine the location of our Q4 DRBD Total course in the United States. So let us know where you would like to see the next incarnation of the industry’s premier Linux High Availability training!
Our survey just takes a minute to complete. We appreciate your input!
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:15:"Florian G. Haas";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:3;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:19:"So Much Work To Do!";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:36:"http://mysql4all.wordpress.com/?p=11";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:61:"http://mysql4all.wordpress.com/2010/08/26/so-much-work-to-do/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:4635:"Few weeks ago I was invited to a dinner in Manchester with a group of CIOs. It was the occasion to talk about Open Source and the use of Open Source Software in the Enterprise.
The conversation went on for quite a while on what is OSS, why OSS is relevant for an Enterprise and how a relatively large organisation can get benefit from OSS. To me, it was time for a sanity check.
When you work in the OSS world, as many of us do, you tend to forget the fears, the doubts and all in all the deep differences between companies who have embraced OSS and others who have not. I am not referring to Linux as a server platform here, since Linux distributions are nowadays recognised as a great server environment and people tend to forget it is OSS. I refer to OSS in general and to the way OSS is used by developers, IT professionals and end users.
Back to my original subject, i.e. the dinner with the CIOs, I will skip the details of a good Brit meal enjoyed by an Italian who is quite fussy when it is a matter of good cuisine, adding the fact that he is a vegetarian. Sticking to the main point, the conversation went on for quite a while on some basic topics. So, the first question I asked the audience was regarding their experience with OSS. Some CIOs mentioned Firefox, some others Linux, others had in mind the last version of a famous media player that could play any kind of media file, contrary to other closed source products.
My main question to the attendees was: what are your feelings about OSS? Well, many attendees were scared by OSS. They relate open source to some blokes in a garage or to a nerd seated all day (after school) in front of a computer in his bedroom. Too much television and blockbusters, I would say. So, where is quality? Where is the stability and robustness of the code?
I tried to lead the attendees to a ground that could be well understood. Take Microsoft for example. I used the Microsoft vs Mozilla classical comparison: which one is more stable, more secure? Would you say that Internet Explorer is more advanced than Firefox? Does it provide a better usability or compatibility? It was easy to find an agreement here and many people revealed that Firefox is their favourite browser, but they did not consider it as an Open Source Software.
There were some other obvious examples, like the stability, performance and scalability provided by Windows Servers vs Linux. “Would you be scared to install a Linux server in your data centre today?”, that was the question. “No, not anymore”, was the answer in unison. So, from the browser, to the server, you can easily move to a more suitable ground, at least for me, which is MySQL. We have estimated 12M+ MySQL server installations and 16 of the 20 most visited web sites use MySQL as their main relational database, probably generating more transactions than the 16 most important banks in the world. Would you be scare to use MySQL as a relational database? The attendees were still not convinced, but at least I instilled some interest.
Another interesting point was raised when we talked about the lifecycle of OSS. Some attendees believed that OSS is more likely to disappear after few months or years, whilst closed source software is more likely to stay around to longer time. So, here is another question for my audience: “have you ever used products like Vignette, Broadvision, Interwoven or Bluemartini?”. Sad faces stared the empty plates, trying to remember the last time they saw the announcement of a new version of what’s left from these famous brands. Then I asked if they heard about projects running on Apache, JBoss, Drupal or Hibernate in their organisations. A mix of different answers revolted the sad faces to cautious smiles. The message was clear: no matter if the software is open or not, the more the software is used, the more it is likely to stay and be adopted for a long time.
At the end of this experience that I wanted to share, one might wonder what is the lesson to learn here. Well, it is pretty simple.
There is so much to do!
This is the reason why I am [still] so excited to work with MySQL. We have an incredible Community, with amazingly skilled contributors. But in my opinion there is a gap, a broken link between OSS and the Enterprise. This gap is particularly visible with MySQL. This is my main focus and where I want to be. Educate the Enterprise to the use of OSS, with focus on MySQL, of course. The more we attract the Enterprise in a virtuous circle, the more this circle can expand and everybody can benefit from this expansion.
So, enjoy the ride, there is more to come!
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 17:29:10 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:13:"Uncategorized";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:6493:"Few weeks ago I was invited to a dinner in Manchester with a group of CIOs. It was the occasion to talk about Open Source and the use of Open Source Software in the Enterprise.
The conversation went on for quite a while on what is OSS, why OSS is relevant for an Enterprise and how a relatively large organisation can get benefit from OSS. To me, it was time for a sanity check.
When you work in the OSS world, as many of us do, you tend to forget the fears, the doubts and all in all the deep differences between companies who have embraced OSS and others who have not. I am not referring to Linux as a server platform here, since Linux distributions are nowadays recognised as a great server environment and people tend to forget it is OSS. I refer to OSS in general and to the way OSS is used by developers, IT professionals and end users.
Back to my original subject, i.e. the dinner with the CIOs, I will skip the details of a good Brit meal enjoyed by an Italian who is quite fussy when it is a matter of good cuisine, adding the fact that he is a vegetarian. Sticking to the main point, the conversation went on for quite a while on some basic topics. So, the first question I asked the audience was regarding their experience with OSS. Some CIOs mentioned Firefox, some others Linux, others had in mind the last version of a famous media player that could play any kind of media file, contrary to other closed source products.
My main question to the attendees was: what are your feelings about OSS? Well, many attendees were scared by OSS. They relate open source to some blokes in a garage or to a nerd seated all day (after school) in front of a computer in his bedroom. Too much television and blockbusters, I would say. So, where is quality? Where is the stability and robustness of the code?
I tried to lead the attendees to a ground that could be well understood. Take Microsoft for example. I used the Microsoft vs Mozilla classical comparison: which one is more stable, more secure? Would you say that Internet Explorer is more advanced than Firefox? Does it provide a better usability or compatibility? It was easy to find an agreement here and many people revealed that Firefox is their favourite browser, but they did not consider it as an Open Source Software.
There were some other obvious examples, like the stability, performance and scalability provided by Windows Servers vs Linux. “Would you be scared to install a Linux server in your data centre today?”, that was the question. “No, not anymore”, was the answer in unison. So, from the browser, to the server, you can easily move to a more suitable ground, at least for me, which is MySQL. We have estimated 12M+ MySQL server installations and 16 of the 20 most visited web sites use MySQL as their main relational database, probably generating more transactions than the 16 most important banks in the world. Would you be scare to use MySQL as a relational database? The attendees were still not convinced, but at least I instilled some interest.
Another interesting point was raised when we talked about the lifecycle of OSS. Some attendees believed that OSS is more likely to disappear after few months or years, whilst closed source software is more likely to stay around to longer time. So, here is another question for my audience: “have you ever used products like Vignette, Broadvision, Interwoven or Bluemartini?”. Sad faces stared the empty plates, trying to remember the last time they saw the announcement of a new version of what’s left from these famous brands. Then I asked if they heard about projects running on Apache, JBoss, Drupal or Hibernate in their organisations. A mix of different answers revolted the sad faces to cautious smiles. The message was clear: no matter if the software is open or not, the more the software is used, the more it is likely to stay and be adopted for a long time.
At the end of this experience that I wanted to share, one might wonder what is the lesson to learn here. Well, it is pretty simple.
There is so much to do!
This is the reason why I am [still] so excited to work with MySQL. We have an incredible Community, with amazingly skilled contributors. But in my opinion there is a gap, a broken link between OSS and the Enterprise. This gap is particularly visible with MySQL. This is my main focus and where I want to be. Educate the Enterprise to the use of OSS, with focus on MySQL, of course. The more we attract the Enterprise in a virtuous circle, the more this circle can expand and everybody can benefit from this expansion.
So, enjoy the ride, there is more to come!
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Ivan Zoratti";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:4;a:6:{s:4:"data";s:108:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:42:"OpenSQL Camp Europe and FrOSCon: A summary";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.lenzg.net/archives/309-guid.html";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:80:"http://www.lenzg.net/archives/309-OpenSQL-Camp-Europe-and-FrOSCon-A-summary.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1870:"With OpenSQL Camp and FrOSCon being over for almost a week now, it's time to come up with a short summary. I traveled home on Monday morning and then took Tuesday off, so I had some catching up to do...As for the past years, FrOSCon rocked again! According to the closing keynote, they had around 1.500 (unique) visitors and I had a great time there. I really enjoyed meeting all the old and new faces of the various Open Source communities. The lineup of speakers was excellent, Jon "maddog" Hall's keynote about "Free and Open Source Software in the Developing World" was quite insightful and inspiring.Most of the time I was busy with speaking at and running the OpenSQL Camp session track in room "HS6", but I managed to sneak out and listen to a few other talks as well. Additionally, I gave a presentation about ZFS on Linux, which had a good crowd and some interesting discussions.This year, all of the sessions in the main lecture halls were streamed live, so people not able to attend could at least get a glimpse on what was going on these two days. Many times it was a really tough decision to make – there were way too many good sessions going in parallel. So being able to see the recording afterwards somewhat relieved that pain. The FrOSCon team has already begun to publish the video files – they are separated by day and lecture room and can be obtained from http://froscon.tv/. And we were really lucky with the weather, too - the weekend was warm and sunny, a perfect setting for the social event, which is one of the highlights of FrOSCon. Impressive: this year, the attendees consumed one cubic meter (1000 liters) of Beer!Finally, I'd like to express my special gratitude to all the speakers of OpenSQL Camp – thank you very much for your support and for devoting your time to participate in our subconference! It was appreciated.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 11:17:27 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:14:{i:0;a:5:{s:4:"data";s:5:"Linux";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:3:"OSS";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:9:"community";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:10:"conference";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:9:"databases";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:5:"event";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:7:"froscon";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:8;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:9;a:5:{s:4:"data";s:11:"opensqlcamp";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:10;a:5:{s:4:"data";s:3:"oss";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:11;a:5:{s:4:"data";s:12:"presentation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:12;a:5:{s:4:"data";s:9:"streaming";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:13;a:5:{s:4:"data";s:3:"zfs";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2645:"With OpenSQL Camp and FrOSCon being over for almost a week now, it's time to come up with a short summary. I traveled home on Monday morning and then took Tuesday off, so I had some catching up to do...
As for the past years, FrOSCon rocked again! According to the closing keynote, they had around 1.500 (unique) visitors and I had a great time there. I really enjoyed meeting all the old and new faces of the various Open Source communities. The lineup of speakers was excellent, Jon "maddog" Hall's keynote about "Free and Open Source Software in the Developing World" was quite insightful and inspiring.
Most of the time I was busy with speaking at and running the OpenSQL Camp session track in room "HS6", but I managed to sneak out and listen to a few other talks as well. Additionally, I gave a presentation about ZFS on Linux, which had a good crowd and some interesting discussions.
This year, all of the sessions in the main lecture halls were streamed live, so people not able to attend could at least get a glimpse on what was going on these two days. Many times it was a really tough decision to make – there were way too many good sessions going in parallel. So being able to see the recording afterwards somewhat relieved that pain. The FrOSCon team has already begun to publish the video files – they are separated by day and lecture room and can be obtained from http://froscon.tv/.
And we were really lucky with the weather, too - the weekend was warm and sunny, a perfect setting for the social event, which is one of the highlights of FrOSCon. Impressive: this year, the attendees consumed one cubic meter (1000 liters) of Beer!
Finally, I'd like to express my special gratitude to all the speakers of OpenSQL Camp – thank you very much for your support and for devoting your time to participate in our subconference! It was appreciated.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Lenz Grimmer";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:5;a:6:{s:4:"data";s:58:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:43:"MySQL Connector/Python and database pooling";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:68:"tag:blogger.com,1999:blog-7603704315097619422.post-15642748880174239";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:79:"http://geert.vanderkelen.org/2010/08/mysql-connectorpython-is-or-should-be.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:899:"MySQL Connector/Python is (or should be) compliant with the Python DB-API 2.0 specification. This means that you can use DBUtils' PooledDB module to implement database connection pooling.Here below you'll find an example which will output the connection ID of each connection requested through the pooling mechanism.from DBUtils.PooledDB import PooledDB
import mysql.connector
def main():
pool_size = 3
pool = PooledDB(mysql.connector, pool_size,
database='test', user='root', host='127.0.0.1')
cnx = [None,] * pool_size
for i in xrange(0,pool_size):
cnx[i] = pool.connection()
cur = cnx[i].cursor()
cur.execute("SELECT CONNECTION_ID()")
print "Cnx %d has ID %d" % (i+1,cur.fetchone()[0])
cur.close()
for c in cnx:
c.close()
The output will be something like this:Cnx 1 has ID 42
Cnx 2 has ID 41
Cnx 3 has ID 40
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 09:12:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:4:{i:0;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"howto";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:6:"python";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:8:"myconnpy";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1501:"MySQL Connector/Python is (or should be) compliant with the Python DB-API 2.0 specification. This means that you can use DBUtils' PooledDB module to implement database connection pooling.
Here below you'll find an example which will output the connection ID of each connection requested through the pooling mechanism.
from DBUtils.PooledDB import PooledDB
import mysql.connector
def main():
pool_size = 3
pool = PooledDB(mysql.connector, pool_size,
database='test', user='root', host='127.0.0.1')
cnx = [None,] * pool_size
for i in xrange(0,pool_size):
cnx[i] = pool.connection()
cur = cnx[i].cursor()
cur.execute("SELECT CONNECTION_ID()")
print "Cnx %d has ID %d" % (i+1,cur.fetchone()[0])
cur.close()
for c in cnx:
c.close()
The output will be something like this:
Cnx 1 has ID 42
Cnx 2 has ID 41
Cnx 3 has ID 40
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:17:"Geert Vanderkelen";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:6;a:6:{s:4:"data";s:63:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:45:"Getting a return code from a stored procedure";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:41:"http://mysqlpreacher.com/wordpress/?p=430";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:89:"http://mysqlpreacher.com/wordpress/2010/08/getting-a-return-code-from-a-stored-procedure/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1619:"Sometimes we have some special need for a stored procedure to call another to do something. That is fine, but what if the second stored proc failed for some reason? Maybe you want to halt the first stored procedure (the caller) and not proceed with the work until the problem is verified and resolved. So How do you do it?
Simple, get a return code from the called procedure to determine if it worked or not!
Here’s a sample piece of code to explain better:
DROP PROCEDURE IF EXISTS `test`.`testing123`;
DELIMITER $$
CREATE
PROCEDURE `test`.`testing123`(OUT a INT)
BEGIN
DECLARE EXIT HANDLER FOR SQLEXCEPTION
BEGIN
SET a=2;
END;
SET a=0;
# toggle the below as comment or not to see the call at the bottom working
# if you uncomment select abc you'll see the error, otherwise all is cool
# select abc;
END$$
DELIMITER ;
DROP PROCEDURE IF EXISTS `test`.`testing456`;
DELIMITER $$
CREATE
PROCEDURE `test`.`testing456`()
BEGIN
SET @a=0;
CALL `test`.`testing123` (@a);
IF @a != 0 THEN
SELECT "There is a problem with proc `testing123`";
ELSE
SELECT "Everything is cool";
END IF;
END$$
DELIMITER ;
CALL `test`.`testing456` ();
testing123 is the second stored proc in this case, called from testing456. The trick is to have an exit handler which returns a status number to the first stored proc (testing456). The latter will hold an if condition to do `something` depending on the return code received by testing123.
If you have any other suggestions I’d appreciate your input.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 06:00:20 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:5:{i:0;a:5:{s:4:"data";s:12:"Intermediate";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:13:"Uncategorized";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:16:"stored procedure";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:17:"stored procedures";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2084:"Sometimes we have some special need for a stored procedure to call another to do something. That is fine, but what if the second stored proc failed for some reason? Maybe you want to halt the first stored procedure (the caller) and not proceed with the work until the problem is verified and resolved. So How do you do it?
Simple, get a return code from the called procedure to determine if it worked or not!
Here’s a sample piece of code to explain better:
DROP PROCEDURE IF EXISTS `test`.`testing123`;
DELIMITER $$
CREATE
PROCEDURE `test`.`testing123`(OUT a INT)
BEGIN
DECLARE EXIT HANDLER FOR SQLEXCEPTION
BEGIN
SET a=2;
END;
SET a=0;
# toggle the below as comment or not to see the call at the bottom working
# if you uncomment select abc you'll see the error, otherwise all is cool
# select abc;
END$$
DELIMITER ;
DROP PROCEDURE IF EXISTS `test`.`testing456`;
DELIMITER $$
CREATE
PROCEDURE `test`.`testing456`()
BEGIN
SET @a=0;
CALL `test`.`testing123` (@a);
IF @a != 0 THEN
SELECT "There is a problem with proc `testing123`";
ELSE
SELECT "Everything is cool";
END IF;
END$$
DELIMITER ;
CALL `test`.`testing456` ();
testing123 is the second stored proc in this case, called from testing456. The trick is to have an exit handler which returns a status number to the first stored proc (testing456). The latter will hold an if condition to do `something` depending on the return code received by testing123.
If you have any other suggestions I’d appreciate your input.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Darren Cassar";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:7;a:6:{s:4:"data";s:58:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:47:"Connector/J ping (there will be a test on this)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:38:"http://mysqlblog.fivefarmers.com/?p=87";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:61:"http://mysqlblog.fivefarmers.com/2010/08/25/connector-j-ping/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2812:"MySQL Connector/J has a useful feature which executes a lightweight ping against a server (or, in the case of load-balanced connections, all active pooled internal connections that are retained) to validate the connection. As you might guess, this feature is useful for Java apps which use connection pools, so that the pool can validate the connection. Depending on your connection pool and configuration, this can be done at different times:
before the pool returns a connection to the application
when the application returns a connection to the pool
during periodic checks of idle connections
So, if you want to use this magic light-weight ping process, here’s how you do it:
Specifying a “validation query” in your connection pool that starts with “/* ping */” _exactly_ will cause the driver to instead send a ping to the server and return a fake result set (much lighter weight), and when using a ReplicationConnection or a LoadBalancedConnection, will send the ping across all active connections.
So says the relevant note in the changelogs.
Now for the test. Which of the following Strings match the condition above?
sql = “/* PING */ SELECT 1″;
sql = “SELECT 1 /* ping*/”;
sql = “/*ping*/ SELECT 1″;
sql = ” /* ping */ SELECT 1″;
sql = “/*to ping or not to ping*/ SELECT 1″;
If you said “none of the above”, you pass the test. The test is sensitive to whitespace, capitalization, and placement. It’s this way for efficiency, as this test is done for every Statement that is executed:
protected static final String PING_MARKER = "/* ping */";
...
if (sql.charAt(0) == '/') {
if (sql.startsWith(PING_MARKER)) {
doPingInstead();
...
All of these statements will issue normal SELECT statements and not get transformed into the lightweight ping. More problematic, this means that loadbalanced connections only execute one statement against one physical connection in the internal pool, rather than validating the connection for each underlying physical connection. So the non-active physical connections sit stale, and stale connections die, and then Connector/J re-balances and selects that stale dead connection, and suddenly you have an Exception pushed up to the application (loadBalanceValidateConnectionOnSwapServer can save you here).
If your Connector/J deployment uses a connection pool which allows you to specify a validation query, check it, and make sure that it starts with “/* ping */” – exactly. This is especially true if you are using load-balancing or replication-aware features of Connector/J – it should help keep alive connections which otherwise will go stale and die, causing problems later.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 04:24:56 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:4:{i:0;a:5:{s:4:"data";s:4:"Java";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:11:"Connector/J";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:14:"Load-balancing";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3372:"MySQL Connector/J has a useful feature which executes a lightweight ping against a server (or, in the case of load-balanced connections, all active pooled internal connections that are retained) to validate the connection. As you might guess, this feature is useful for Java apps which use connection pools, so that the pool can validate the connection. Depending on your connection pool and configuration, this can be done at different times:
- before the pool returns a connection to the application
- when the application returns a connection to the pool
- during periodic checks of idle connections
So, if you want to use this magic light-weight ping process, here’s how you do it:
Specifying a “validation query” in your connection pool that starts with “/* ping */” _exactly_ will cause the driver to instead send a ping to the server and return a fake result set (much lighter weight), and when using a ReplicationConnection or a LoadBalancedConnection, will send the ping across all active connections.
So says the relevant note in the changelogs.
Now for the test. Which of the following Strings match the condition above?
- sql = “/* PING */ SELECT 1″;
- sql = “SELECT 1 /* ping*/”;
- sql = “/*ping*/ SELECT 1″;
- sql = ” /* ping */ SELECT 1″;
- sql = “/*to ping or not to ping*/ SELECT 1″;
If you said “none of the above”, you pass the test. The test is sensitive to whitespace, capitalization, and placement. It’s this way for efficiency, as this test is done for every Statement that is executed:
protected static final String PING_MARKER = "/* ping */";
...
if (sql.charAt(0) == '/') {
if (sql.startsWith(PING_MARKER)) {
doPingInstead();
...
All of these statements will issue normal SELECT statements and not get transformed into the lightweight ping. More problematic, this means that loadbalanced connections only execute one statement against one physical connection in the internal pool, rather than validating the connection for each underlying physical connection. So the non-active physical connections sit stale, and stale connections die, and then Connector/J re-balances and selects that stale dead connection, and suddenly you have an Exception pushed up to the application (loadBalanceValidateConnectionOnSwapServer can save you here).
If your Connector/J deployment uses a connection pool which allows you to specify a validation query, check it, and make sure that it starts with “/* ping */” – exactly. This is especially true if you are using load-balancing or replication-aware features of Connector/J – it should help keep alive connections which otherwise will go stale and die, causing problems later.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:11:"Todd Farmer";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:8;a:6:{s:4:"data";s:63:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:53:"Monty Program Ab All Company Meeting in Istanbul 2010";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:48:"http://esslingen.homeunix.net/~hakan/blog/?p=275";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:107:"http://esslingen.homeunix.net/~hakan/blog/2010/08/26/monty-program-ab-all-company-meeting-in-istanbul-2010/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1317:"As you might know, we are a completely virtual company and we only meet once or twice a year in real life. This year’s decision on where to meet is Istanbul. Being the only Turkish citizen born in Germany with Georgian ancestors in the company, I was asked to organize our meeting.
My first step was to look for excellence: Kaj Arnö’s blog post about How to arrange a physical meeting in a virtual organisation.
We first conducted an internal poll about the meeting date and decided on meeting in Istanbul in October. Specifically, from Thursday, October 7th until Tuesday, October 12th 2010.
Organizing a meeting for around 30 people is not a trivial task, therefore I got an assistant — a quite famous and popular one to be be frank — My Widenius joined me for preparing our meeting. My is the My in MySQL and she has an askmonty.org email address now, too.
If you will be in Istanbul at the beginning of October, please feel free to drop us a note and we will be happy to invite you to one of our meetings. We will have three meetings days: Friday, Saturday, and Sunday. The majority of the meetings each day will be public. We already have guests in our queue from Facebook, Percona, and Intel.
We are also looking for local (Istanbul) user groups and individuals who have interest in meeting us.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 26 Aug 2010 03:13:37 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:5:{i:0;a:5:{s:4:"data";s:7:"MariaDB";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:10:"conference";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:13:"Istanbul 2010";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:7:"meeting";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2213:"As you might know, we are a completely virtual company and we only meet once or twice a year in real life. This year’s decision on where to meet is Istanbul. Being the only Turkish citizen born in Germany with Georgian ancestors in the company, I was asked to organize our meeting.
My first step was to look for excellence: Kaj Arnö’s blog post about How to arrange a physical meeting in a virtual organisation.
We first conducted an internal poll about the meeting date and decided on meeting in Istanbul in October. Specifically, from Thursday, October 7th until Tuesday, October 12th 2010.
Organizing a meeting for around 30 people is not a trivial task, therefore I got an assistant — a quite famous and popular one to be be frank — My Widenius joined me for preparing our meeting. My is the My in MySQL and she has an askmonty.org email address now, too.
If you will be in Istanbul at the beginning of October, please feel free to drop us a note and we will be happy to invite you to one of our meetings. We will have three meetings days: Friday, Saturday, and Sunday. The majority of the meetings each day will be public. We already have guests in our queue from Facebook, Percona, and Intel.
We are also looking for local (Istanbul) user groups and individuals who have interest in meeting us.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:17:"Hakan Kucukyilmaz";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:9;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:13:"Multi Slicing";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:36:"http://www.rethinkdb.com/blog/?p=500";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:52:"http://www.rethinkdb.com/blog/2010/08/multi-slicing/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2733:"The basic data structure that powers databases is called a B-tree. This is where you actually store the user’s data. B-trees are great because you can put huge amounts of data in them and access remains fast. In fact, the naive strategy of putting a whole data set into one B-Tree doesn’t break down because of access time. It does, however, break down when you try to support a multiaccess paradigm.
Six years ago, multiaccess was nice. Now that processors have multiple cores, it’s crucial. Four cores fighting over one B-tree means a lot of wasted processor time. In a multiaccess scheme, different cores can concurrently access data. This gets tricky. You can go looking for a piece of data only to find that someone has moved it since you started; that’s trouble: for all you know it was deleted. You could start the search over, but without guarantees—maybe you’ll get unlucky and it will be plucked out from under you again. How do you know when to give up? Your database is now blazingly fast, but also broken. We handle this with a locking scheme.
With a locking scheme, you gain the ability to say: “Okay, I’m going to go look for this piece of data, no one else is allowed to touch it.” If someone else wants to get at that piece of data before you’re done—they have to wait. The database is fixed, but it’s not quite as blazingly fast as it used to be. The problem is that sometimes cores have to wait for data, and while they wait they’re just twiddling their thumbs. There isn’t a good way to prevent these waits, but we can put that time toward something productive.
The problem isn’t the waiting, it’s the lack of things to do during that time. The solution is to slice up our B-Tree into several smaller B-Trees. Every possible key has one and only one slice that it can ever be stored in. We do this with hashing. Now even if a core locks up an entire B-Tree, you can still do work in the intervening time.
Now that we support multiple slices, the question is: how many slices do we actually want? There are downsides to both extremes: too many slices and the efficiency of the B-Tree structure is underused, too few and CPUs are underused. At RethinkDB we’ve found experimentally that 4 is a sufficiently high value. The quick back-of-the-envelope calculation is that an unsliced B-Tree will defy computation about of the time. That translates to an hour of wasted CPU time every 2 days. This waste falls off very sharply with our innovation: 2 slices are busy: (just of the time) and 4 slices are busy or . At which point it will take you 700 years to waste an hour of CPU time. That’s a number we can live with. ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Wed, 25 Aug 2010 17:00:43 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:13:"Announcements";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3775:"The basic data structure that powers databases is called a B-tree. This is where you actually store the user’s data. B-trees are great because you can put huge amounts of data in them and access remains fast. In fact, the naive strategy of putting a whole data set into one B-Tree doesn’t break down because of access time. It does, however, break down when you try to support a multiaccess paradigm.
Six years ago, multiaccess was nice. Now that processors have multiple cores, it’s crucial. Four cores fighting over one B-tree means a lot of wasted processor time. In a multiaccess scheme, different cores can concurrently access data. This gets tricky. You can go looking for a piece of data only to find that someone has moved it since you started; that’s trouble: for all you know it was deleted. You could start the search over, but without guarantees—maybe you’ll get unlucky and it will be plucked out from under you again. How do you know when to give up? Your database is now blazingly fast, but also broken. We handle this with a locking scheme.
With a locking scheme, you gain the ability to say: “Okay, I’m going to go look for this piece of data, no one else is allowed to touch it.” If someone else wants to get at that piece of data before you’re done—they have to wait. The database is fixed, but it’s not quite as blazingly fast as it used to be. The problem is that sometimes cores have to wait for data, and while they wait they’re just twiddling their thumbs. There isn’t a good way to prevent these waits, but we can put that time toward something productive.
The problem isn’t the waiting, it’s the lack of things to do during that time. The solution is to slice up our B-Tree into several smaller B-Trees. Every possible key has one and only one slice that it can ever be stored in. We do this with hashing. Now even if a core locks up an entire B-Tree, you can still do work in the intervening time.
Now that we support multiple slices, the question is: how many slices do we actually want? There are downsides to both extremes: too many slices and the efficiency of the B-Tree structure is underused, too few and CPUs are underused. At RethinkDB we’ve found experimentally that 4 is a sufficiently high value. The quick back-of-the-envelope calculation is that an unsliced B-Tree will defy computation about of the time. That translates to an hour of wasted CPU time every 2 days. This waste falls off very sharply with our innovation: 2 slices are busy: (just of the time) and 4 slices are busy or . At which point it will take you 700 years to waste an hour of CPU time. That’s a number we can live with.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:15:"Slava Akhmechet";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:10;a:6:{s:4:"data";s:38:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:5:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:35:"Querying for InnoDB Lock contention";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:70:"tag:blogger.com,1999:blog-9144505959002328789.post-1086182254653935275";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:88:"http://karlssonondatabases.blogspot.com/2010/08/querying-for-innodb-lock-contention.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1259:"In the previous post, I showed how the "virtual tables" that the InnoDB Plugin use for finding lock contention works, and what they look like. I also showed what an InnoDB lock graph looks will look like when monitored from MyQuery 3.3, which is not yet, but soon, released.So, we are looking at three tables, the transaction table, where each transaction has no, one or more locks. For our intents and purposes, we will disregard transactions without locks, so we can join transactions table to the locks table. Then we need to link up the locks table to the lock waiters, to see if the lock in question is waiting on another lock, Each lock is either not waiting on one, or, as we have seen, more, locks, or it is not waiting at all. As a lock might not be waiting at all, we need an OUTER join here.So the query I am using, the result of which is massaged a fair bit before I shown the lock tree, but this is the query I use to get the data:SELECT t.trx_id, l.lock_id, w.blocking_lock_id, l.lock_table, l.lock_index,t.trx_query, l.lock_data FROM information_schema.innodb_trx AS t JOIN information_schema.innodb_locks AS l ON t.trx_id = l.lock_trx_id LEFT OUTER JOIN information_schema.innodb_lock_waits AS w ON l.lock_id = w.requested_lock_id/Karlsson";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Wed, 25 Aug 2010 15:13:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1950:"In the previous post, I showed how the "virtual tables" that the InnoDB Plugin use for finding lock contention works, and what they look like. I also showed what an InnoDB lock graph looks will look like when monitored from MyQuery 3.3, which is not yet, but soon, released.
So, we are looking at three tables, the transaction table, where each transaction has no, one or more locks. For our intents and purposes, we will disregard transactions without locks, so we can join transactions table to the locks table. Then we need to link up the locks table to the lock waiters, to see if the lock in question is waiting on another lock, Each lock is either not waiting on one, or, as we have seen, more, locks, or it is not waiting at all. As a lock might not be waiting at all, we need an OUTER join here.
So the query I am using, the result of which is massaged a fair bit before I shown the lock tree, but this is the query I use to get the data:
SELECT t.trx_id, l.lock_id, w.blocking_lock_id, l.lock_table, l.lock_index,
t.trx_query, l.lock_data
FROM information_schema.innodb_trx AS t
JOIN information_schema.innodb_locks AS l ON t.trx_id = l.lock_trx_id
LEFT OUTER JOIN information_schema.innodb_lock_waits AS w
ON l.lock_id = w.requested_lock_id
/Karlsson
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:15:"Anders Karlsson";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:11;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:24:"Multi-master replication";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:38:"http://thenoyes.com/littlenoise/?p=117";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:38:"http://thenoyes.com/littlenoise/?p=117";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2351:"Implement multi-master to single-slave replication by periodically saving state and changing masters, and do it all in SQL.*
Just a proof of concept. I suggest you add some error checking (like, don't make the switch if the slave isn't running).
Remember that all this stuff goes on the SLAVE. If you absent-mindedly log in to the master, like I did a few times, you might wonder why your relay_log.info file isn't read.
Set up a table to keep track of each master file. Mine use the same user name and password for all hosts; you can add those fields and adjust the procedure accordingly if you like.
USE mysql;
DROP TABLE IF EXISTS rotate_master;
CREATE TABLE rotate_master (
id int auto_increment primary key,
master_host varchar(255),
master_port int unsigned,
master_log_file varchar(255),
master_log_pos int unsigned,
in_use boolean DEFAULT 0
);
Insert the list of masters. The one that is currently master should have `in_use` set to 1; all the others should be 0. The order you insert them is the order they will rotate.
INSERT INTO rotate_master VALUES (NULL, 'localhost', 3306, 'bin.000001', 0, 1);
INSERT INTO rotate_master VALUES (NULL, 'localhost', 3308, 'bin.000001', 0, 0);
Then, a procedure to grab the current position (by reading from the relay_log.info file, since the slave position is not otherwise exposed in any accessible manner), record it in the above table, and change masters to the next one in the list.
DROP PROCEDURE IF EXISTS rotate_master;
DELIMITER //
CREATE PROCEDURE rotate_master()
BEGIN
DECLARE _info text;
DECLARE _master_file varchar(255);
DECLARE _master_pos int unsigned;
DECLARE _master_host varchar(255);
DECLARE _master_port int unsigned;
DECLARE _id int;
STOP SLAVE;
-- fetch and store current position
SELECT LOAD_FILE(@@relay_log_info_file) INTO _info;
SELECT
SUBSTRING_INDEX(SUBSTRING_INDEX(_info, '\n', 3), '\n', -1),
SUBSTRING_INDEX(SUBSTRING_INDEX(_info, '\n', 4), '\n', -1)
INTO _master_file, _master_pos;
UPDATE mysql.rotate_master SET master_log_file = _master_file, master_log_pos = _master_pos, id = LAST_INSERT_ID(id) WHERE in_use = 1;
-- fetch next host
SELECT
id,
master_host,
master_port,
master_log_file,
master_log_pos
INTO _id, _master_host, _master_port, _master_file, _master_pos
FROM rotate_master
ORDER BY id ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Wed, 25 Aug 2010 14:35:12 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3865:"Implement multi-master to single-slave replication by periodically saving state and changing masters, and do it all in SQL.*
Just a proof of concept. I suggest you add some error checking (like, don't make the switch if the slave isn't running).
Remember that all this stuff goes on the SLAVE. If you absent-mindedly log in to the master, like I did a few times, you might wonder why your relay_log.info file isn't read.
Set up a table to keep track of each master file. Mine use the same user name and password for all hosts; you can add those fields and adjust the procedure accordingly if you like.
USE mysql;
DROP TABLE IF EXISTS rotate_master;
CREATE TABLE rotate_master (
id int auto_increment primary key,
master_host varchar(255),
master_port int unsigned,
master_log_file varchar(255),
master_log_pos int unsigned,
in_use boolean DEFAULT 0
);
Insert the list of masters. The one that is currently master should have `in_use` set to 1; all the others should be 0. The order you insert them is the order they will rotate.
INSERT INTO rotate_master VALUES (NULL, 'localhost', 3306, 'bin.000001', 0, 1);
INSERT INTO rotate_master VALUES (NULL, 'localhost', 3308, 'bin.000001', 0, 0);
Then, a procedure to grab the current position (by reading from the relay_log.info file, since the slave position is not otherwise exposed in any accessible manner), record it in the above table, and change masters to the next one in the list.
DROP PROCEDURE IF EXISTS rotate_master;
DELIMITER //
CREATE PROCEDURE rotate_master()
BEGIN
DECLARE _info text;
DECLARE _master_file varchar(255);
DECLARE _master_pos int unsigned;
DECLARE _master_host varchar(255);
DECLARE _master_port int unsigned;
DECLARE _id int;
STOP SLAVE;
-- fetch and store current position
SELECT LOAD_FILE(@@relay_log_info_file) INTO _info;
SELECT
SUBSTRING_INDEX(SUBSTRING_INDEX(_info, '\n', 3), '\n', -1),
SUBSTRING_INDEX(SUBSTRING_INDEX(_info, '\n', 4), '\n', -1)
INTO _master_file, _master_pos;
UPDATE mysql.rotate_master SET master_log_file = _master_file, master_log_pos = _master_pos, id = LAST_INSERT_ID(id) WHERE in_use = 1;
-- fetch next host
SELECT
id,
master_host,
master_port,
master_log_file,
master_log_pos
INTO _id, _master_host, _master_port, _master_file, _master_pos
FROM rotate_master
ORDER BY id <= LAST_INSERT_ID(), id LIMIT 1;
-- advance to next host
SET @sql := CONCAT(
'CHANGE MASTER TO master_host=', QUOTE(_master_host),
', master_port=', _master_port,
', master_log_file=', QUOTE(_master_file),
', master_log_pos=', _master_pos);
PREPARE myStmt FROM @sql;
EXECUTE myStmt;
-- mark host as changed
UPDATE mysql.rotate_master SET in_use = 0 WHERE in_use = 1;
UPDATE mysql.rotate_master SET in_use = 1 WHERE id = _id;
START SLAVE;
END//
DELIMITER ;
Finally, schedule it to rotate. I used 10 seconds just to play with; I imagine a few minutes would be more practical. *If you don't have 5.1 or later, you'll have to schedule this part externally, with cron or task scheduler.
DROP EVENT IF EXISTS rotate_master;
CREATE EVENT rotate_master
ON SCHEDULE EVERY 10 SECOND
DO CALL mysql.rotate_master();
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:11:"Scott Noyes";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:12;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:33:"Monitoring InnoDB Lock contention";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:70:"tag:blogger.com,1999:blog-9144505959002328789.post-4016905035317322238";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:86:"http://karlssonondatabases.blogspot.com/2010/08/monitoring-innodb-lock-contention.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:5691:"When you use InnoDB, or any other RDBMS or storage engine that supports row level locking and transactions, you get to realize a few things:The chance of lock contention is less, as the lock is much more granual that page or table level locking.On the other hand, when you get into lock contention, maybe because you have structured your data in such a way so there may well be as many locks as before (for example if one small table with very few rows is updated frequently by many threads), then the effect is worse.With transactions, which are a good thing by the way, I'm not promoting not transactions datastores here and assuming auto commit isn't used, then locks are held longer, and as each transaction may well hold several locks, because of multiple rows being updated or multiple statements are part of one transaction for example.With InnoDB then, fining the root cause when you end up with lock contention, with several transactions locking each other out in a complex manner, is HARD!Luckily, finding locks and waiters for them, and the relationship between these, is A LOT easier when you use the InnoDB Plugin instead of the builtin InnoDB (are you still using the built-in? Dont. Just don't! The plugin has better performance, is easier to monitor, has online-operations and is GA, even in 5.1, you just enable it with a few settings and you are ready to go).With the InnoDB Plugin and the InnoDB Plugin Information schema tables installed, the latter by installing the plugins using the plugin_load my.ini setting, for example:plugin_load="innodb=ha_innodb_plugin.dll;innodb_trx=ha_innodb_plugin.dll;innodb_locks=ha_innodb_plugin.dll;innodb_lock_waits=ha_innodb_plugin.dll"we are ready to go.There are 3 new tables in the INFORMATION_SCHEMA schema now, one for each object we need for this exercise:INNODB_TRX - This keeps track of the InnoDB transactions. Each transaction may have no, one or more locks and may also be waiting on locks (locks I say? What? More about that later)INNODB_LOCKS - This keeps track of all locks for all transactions. Locks are unique for transactions, so if two transactions are waiting for one lock, we will see a total of three locks, one holding the lock, and two other locks, for the same row, waiting for it.INNODB_LOCK_WAITS - This table has one row for each lock that is waiting for another lock.In InnoDB, only locks that are actually interesting are show in these tables, so if a transactions holds a lock that noone is waiting for, this lock will not show up.The are some intersting things to note about these tables. If we have 3 transactions:A holds a lock on a rowB is waiting for the lock A holds.C is waiting for the same lock that A holds and that B is waiting for.We will see 3 locks! Why? A holds a lock, right, so that is fine. B holds a lock and is waiting for A, so that is fine too. But does C hold a lock? Yes, it holds a lock and s waiting for the one that B holds. I told you that locks, as seen in this table, are unqiue to each transaction.If we now assume that A just holds the lock, and isnät waiting on anything, we have how many waiters? Well, we have 2 waiting transactions, but 3 waiters! Transaction C, to complete, needs both the lock that A holds AND the lock that B holds (as we saw above, just because B is waiting for a lock on a row, doesn't maen it can also HOLD a row on that row. A different loc, but the same row). B waits for A and C waits for A and B, whch menas we have a total of three waiters! But if you look at the individual transactions, as represented by the requesting_trx_id column in the INNODB_LOCK_WAITS table, we see only two unique transactions, which is right, as although we have 3 lock waiting, only 2 transactions are actually blocked (B and C). And if we look at the column blocking_trx_id column, then we again see only 2 unique transactions blocking something, A and B this time, as C isn't really blocking something!The actual data may look something like this, in this example:SELECT * FROM innodb_locks;lock_id lock_trx_id lock_mode lock_type lock_table lock_index lock_space lock_page lock_rec lock_dataB87:0:52:6 B87 X RECORD `test`.`t1` `PRIMARY` 0 52 6 5B86:0:52:6 B86 X RECORD `test`.`t1` `PRIMARY` 0 52 6 5B85:0:52:6 B85 X RECORD `test`.`t1` `PRIMARY` 0 52 6 5And the the waiters:SELECT * FROM innodb_lock_waits;requesting_trx_id requested_lock_id blocking_trx_id blocking_lock_idB87 B87:0:52:6 B86 B86:0:52:6B87 B87:0:52:6 B85 B85:0:52:6B86 B86:0:52:6 B85 B85:0:52:6One interesting column that has not been discussed so far is the column lock_data in the INNODB_LOCKS table. This shows the value that is being locked, in this case I have an integer column with the value 5 (and this is the PRIMARY KEY also, as you can see from the lock_index column).Does this help? Yes, it is much better than using the built-in InnoDB, but there is still some way to go. A simple SELECT from these tables will be a good help, but as this data is actually in a hierarchy, it would be real nice to see this as a lock graph? Right? Well that is something I am working on for MyQuery 3.3, which is due soon. It will look something like this:This is the same lock tree as in the example above, and how it will look, but there is something that is not right here, right? The C lock is just shown as waiting on A not B? Well, for the sake of clarity, I have cleaned that up in the GUI itself, so only relevant locks and waits are really shown./Karlsson";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Wed, 25 Aug 2010 12:27:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:33:"mysql plugin innodb locks myquery";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:7054:"When you use InnoDB, or any other RDBMS or storage engine that supports row level locking and transactions, you get to realize a few things:- The chance of lock contention is less, as the lock is much more granual that page or table level locking.
- On the other hand, when you get into lock contention, maybe because you have structured your data in such a way so there may well be as many locks as before (for example if one small table with very few rows is updated frequently by many threads), then the effect is worse.
- With transactions, which are a good thing by the way, I'm not promoting not transactions datastores here and assuming auto commit isn't used, then locks are held longer, and as each transaction may well hold several locks, because of multiple rows being updated or multiple statements are part of one transaction for example.
- With InnoDB then, fining the root cause when you end up with lock contention, with several transactions locking each other out in a complex manner, is HARD!
Luckily, finding locks and waiters for them, and the relationship between these, is A LOT easier when you use the InnoDB Plugin instead of the builtin InnoDB (are you still using the built-in? Dont. Just don't! The plugin has better performance, is easier to monitor, has online-operations and is GA, even in 5.1, you just enable it with a few settings and you are ready to go).
With the InnoDB Plugin and the InnoDB Plugin Information schema tables installed, the latter by installing the plugins using the plugin_load my.ini setting, for example:plugin_load="innodb=ha_innodb_plugin.dll;innodb_trx=ha_innodb_plugin.dll;innodb_locks=ha_innodb_plugin.dll;innodb_lock_waits=ha_innodb_plugin.dll"
we are ready to go.
There are 3 new tables in the INFORMATION_SCHEMA schema now, one for each object we need for this exercise:
- INNODB_TRX - This keeps track of the InnoDB transactions. Each transaction may have no, one or more locks and may also be waiting on locks (locks I say? What? More about that later)
- INNODB_LOCKS - This keeps track of all locks for all transactions. Locks are unique for transactions, so if two transactions are waiting for one lock, we will see a total of three locks, one holding the lock, and two other locks, for the same row, waiting for it.
- INNODB_LOCK_WAITS - This table has one row for each lock that is waiting for another lock.
In InnoDB, only locks that are actually interesting are show in these tables, so if a transactions holds a lock that noone is waiting for, this lock will not show up.
The are some intersting things to note about these tables. If we have 3 transactions:
- A holds a lock on a row
- B is waiting for the lock A holds.
- C is waiting for the same lock that A holds and that B is waiting for.
We will see 3 locks! Why? A holds a lock, right, so that is fine. B holds a lock and is waiting for A, so that is fine too. But does C hold a lock? Yes, it holds a lock and s waiting for the one that B holds. I told you that locks, as seen in this table, are unqiue to each transaction.
If we now assume that A just holds the lock, and isnät waiting on anything, we have how many waiters? Well, we have 2 waiting transactions, but 3 waiters! Transaction C, to complete, needs both the lock that A holds AND the lock that B holds (as we saw above, just because B is waiting for a lock on a row, doesn't maen it can also HOLD a row on that row. A different loc, but the same row). B waits for A and C waits for A and B, whch menas we have a total of three waiters! But if you look at the individual transactions, as represented by the requesting_trx_id column in the INNODB_LOCK_WAITS table, we see only two unique transactions, which is right, as although we have 3 lock waiting, only 2 transactions are actually blocked (B and C). And if we look at the column blocking_trx_id column, then we again see only 2 unique transactions blocking something, A and B this time, as C isn't really blocking something!
The actual data may look something like this, in this example:
SELECT * FROM innodb_locks;
lock_id lock_trx_id lock_mode lock_type lock_table lock_index lock_space lock_page lock_rec lock_data
B87:0:52:6 B87 X RECORD `test`.`t1` `PRIMARY` 0 52 6 5
B86:0:52:6 B86 X RECORD `test`.`t1` `PRIMARY` 0 52 6 5
B85:0:52:6 B85 X RECORD `test`.`t1` `PRIMARY` 0 52 6 5
And the the waiters:SELECT * FROM innodb_lock_waits;
requesting_trx_id requested_lock_id blocking_trx_id blocking_lock_id
B87 B87:0:52:6 B86 B86:0:52:6
B87 B87:0:52:6 B85 B85:0:52:6
B86 B86:0:52:6 B85 B85:0:52:6
One interesting column that has not been discussed so far is the column lock_data in the INNODB_LOCKS table. This shows the value that is being locked, in this case I have an integer column with the value 5 (and this is the PRIMARY KEY also, as you can see from the lock_index column).
Does this help? Yes, it is much better than using the built-in InnoDB, but there is still some way to go. A simple SELECT from these tables will be a good help, but as this data is actually in a hierarchy, it would be real nice to see this as a lock graph? Right? Well that is something I am working on for MyQuery 3.3, which is due soon. It will look something like this:
This is the same lock tree as in the example above, and how it will look, but there is something that is not right here, right? The C lock is just shown as waiting on A not B? Well, for the sake of clarity, I have cleaned that up in the GUI itself, so only relevant locks and waits are really shown.
/Karlsson
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:15:"Anders Karlsson";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:13;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:53:"distributed pushed down joins - progress and attitude";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:70:"tag:blogger.com,1999:blog-1526532204016125586.post-1164025800112636889";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:84:"http://jonasoreland.blogspot.com/2010/08/distributed-pushed-down-joins-progress.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:917:"we're now at a stage so we can test/benchmark scan/scan queries, i.e push most queries. there are still know bugs and things that we can't push.to see how we performs, we tested a very old query/dataset, that a customer tried in the 4.1 days. the result at that time was a disaster, and they at that time decided to use something else.it's a 11-way join with mainly REF accesses (i.e not many lookups)i tested it on same hardware/configuration as for my UC presentation.without join pushdown the average query time is 3.6s.with join pushdown it's ~500ms, i.e ~7x improvement.the nights of kni are deeply disappointed, and that's an attitude i like :-)FYI: we also optimized TPCW::getBestSeller some more and are now at 42x on that hardware/configuration.---stay tuned---micro update: Found a bug when using ndbmtd...fixing that gave 200ms (i.e 18x)...also did some more optimizations and now record is 120ms (i.e 29x)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Wed, 25 Aug 2010 09:46:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1597:"we're now at a stage so we can test/benchmark scan/scan queries, i.e push most queries. there are still know bugs and things that we can't push.
to see how we performs, we tested a very old query/dataset, that a customer tried in the 4.1 days. the result at that time was a disaster, and they at that time decided to use something else.
it's a 11-way join with mainly REF accesses (i.e not many lookups)
i tested it on same hardware/configuration as for my UC presentation.
without join pushdown the average query time is 3.6s.
with join pushdown it's ~500ms, i.e ~7x improvement.
the nights of kni are deeply disappointed, and that's an attitude i like :-)
FYI: we also optimized TPCW::getBestSeller some more and are now at 42x on that hardware/configuration.
---
stay tuned
---
micro update: Found a bug when using ndbmtd...fixing that gave 200ms (i.e 18x)...also did some more optimizations and now record is 120ms (i.e 29x)
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Jonas Oreland";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:14;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:36:"OpenSQLCamp Boston seeking donations";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:31:"1079 at http://technocation.org";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:68:"http://technocation.org/content/opensqlcamp-boston-seeking-donations";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:609:"-----------
If you are not familiar with OpenSQLCamp, here's a description from the home page at http://opensqlcamp.org/:
OpenSQL Camp is open to all – sessions have included PostgreSQL, SQLite, MySQL (and storage engines and forks thereof), Drizzle (and tools such as Google Proto Buffers), DBIx::Cache, Gearman, cloud computing, Unix tips, query optimization, Apache Derby, BlackRay, Continuent Tungsten, DB Clustering, Firebird, CouchDB, MongoDB, Cassandra, Firewater, how to version schemas, Waffle Grid / Storm Cloud, databases on SSDs, and even "soft" topics like the Open Database Alliance.
read more";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Wed, 25 Aug 2010 06:10:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:11:"Conferences";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:12:"You Can Help";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:970:"-----------
If you are not familiar with OpenSQLCamp, here's a description from the home page at http://opensqlcamp.org/:
OpenSQL Camp is open to all – sessions have included PostgreSQL, SQLite, MySQL (and storage engines and forks thereof), Drizzle (and tools such as Google Proto Buffers), DBIx::Cache, Gearman, cloud computing, Unix tips, query optimization, Apache Derby, BlackRay, Continuent Tungsten, DB Clustering, Firebird, CouchDB, MongoDB, Cassandra, Firewater, how to version schemas, Waffle Grid / Storm Cloud, databases on SSDs, and even "soft" topics like the Open Database Alliance.
read more
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Technocation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:15;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:71:"How to do user language/locale detection quickly without Zend Framework";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:27:"http://www.mpopp.net/?p=159";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:100:"http://www.mpopp.net/2010/08/how-to-do-user-languagelocale-detection-quickly-without-zend-framework/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2109:"Recently I wrote about detecting the preferred language or locale of a web site visitor, using Zend Framework.
Well, I have to start with one correction. In my last blog post about this topic I talked about the User Agent String, but since I wrote the article, I figured out that the User Agent String doesn’t play any role at all. Neither in the Zend Framework variant that I blogged about earlier, nor in today’s code. And that is good so, because both Internet Explorer and Mozilla Firefox will no longer add the browser language or locale to their User Agent Strings in the future. Other browsers are likely to follow, since there are efforts to make it harder to fingerprint users by their UA-Strings.
So here is my ZF-less variant:
$m = array();
$http_accept_language =
isset($_SERVER['HTTP_ACCEPT_LANGUAGE']) ?
$_SERVER['HTTP_ACCEPT_LANGUAGE'] :
"";
preg_match_all('/([a-z]{2})(?:-[a-zA-Z]{2})?/',
$http_accept_language,
$m);
$pref_locale = array();
foreach($m[0] as $locale) {
$pref_locale[] = $locale;
}
$pref_lang = array();
foreach($m[1] as $lang) {
if (! in_array($lang, $pref_lang)) {
$pref_lang[] = $lang;
}
}
The array $pref_locale will contain all locales in order of the user’s preference, and $pref_lang will only contain the languages in order of the user’s preference. Other than my ZF code from last time, this allows to look up secondary, tertiary etc. choices of the user as well.
Here is an example. Lets assume, these are the user’s preference settings:
$_SERVER['HTTP_ACCEPT_LANGUAGE'] would contain:
en-us,en;q=0.9,de-at;q=0.7,de;q=0.6,es-es;q=0.4,es;q=0.3,ja;q=0.1.
After running my code, the $pref_locale array would contain this:
Array
(
[0] => en-us
[1] => en
[2] => de-at
[3] => de
[4] => es-es
[5] => es
[6] => ja
)
and the $pref_lang array would contain
Array
(
[0] => en
[1] => de
[2] => es
[3] => ja
)
Still simple enough, and most importantly, still a better solution than what Google still does.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 23:39:20 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:11:"PlanetMySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:3:"WWW";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3187:"Recently I wrote about detecting the preferred language or locale of a web site visitor, using Zend Framework.
Well, I have to start with one correction. In my last blog post about this topic I talked about the User Agent String, but since I wrote the article, I figured out that the User Agent String doesn’t play any role at all. Neither in the Zend Framework variant that I blogged about earlier, nor in today’s code. And that is good so, because both Internet Explorer and Mozilla Firefox will no longer add the browser language or locale to their User Agent Strings in the future. Other browsers are likely to follow, since there are efforts to make it harder to fingerprint users by their UA-Strings.
So here is my ZF-less variant:
$m = array();
$http_accept_language =
isset($_SERVER['HTTP_ACCEPT_LANGUAGE']) ?
$_SERVER['HTTP_ACCEPT_LANGUAGE'] :
"";
preg_match_all('/([a-z]{2})(?:-[a-zA-Z]{2})?/',
$http_accept_language,
$m);
$pref_locale = array();
foreach($m[0] as $locale) {
$pref_locale[] = $locale;
}
$pref_lang = array();
foreach($m[1] as $lang) {
if (! in_array($lang, $pref_lang)) {
$pref_lang[] = $lang;
}
}
The array $pref_locale
will contain all locales in order of the user’s preference, and $pref_lang
will only contain the languages in order of the user’s preference. Other than my ZF code from last time, this allows to look up secondary, tertiary etc. choices of the user as well.
Here is an example. Lets assume, these are the user’s preference settings:
$_SERVER['HTTP_ACCEPT_LANGUAGE']
would contain:
en-us,en;q=0.9,de-at;q=0.7,de;q=0.6,es-es;q=0.4,es;q=0.3,ja;q=0.1.
After running my code, the $pref_locale
array would contain this:
Array
(
[0] => en-us
[1] => en
[2] => de-at
[3] => de
[4] => es-es
[5] => es
[6] => ja
)
and the $pref_lang
array would contain
Array
(
[0] => en
[1] => de
[2] => es
[3] => ja
)
Still simple enough, and most importantly, still a better solution than what Google still does.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:11:"Markus Popp";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:16;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:24:"20 latest unique records";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:34:"http://explainextended.com/?p=4884";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:63:"http://explainextended.com/2010/08/24/20-latest-unique-records/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:14362:"From Stack Overflow:
I have a logfile which logs the insert/delete/updates from all kinds of tables.
I would like to get an overview of for example the last 20 people which records where updated, ordered by the last update (datetime DESC)
A common solution for such a task would be writing an aggregate query with ORDER BY and LIMIT:
SELECT person, MAX(ts) AS last_update
FROM logfile
GROUP BY
person
ORDER BY
last_update DESC
LIMIT 20
What’s bad in this solution? Performance, as usual.
Since last_update is an aggregate, it cannot be indexed. And ORDER BY on unindexed fields results in our good old friend, filesort.
Note that even in this case the indexes can be used and the full table scan can be avoided: if there is an index on (person, ts), MySQL will tend to use a loose index scan on this index, which can save this query if there are relatively few persons in the table. However, if there are many (which is what we can expect for a log table), loose index scan can even degrade performance and generally will be avoided by MySQL.
We should use another approach here. Let’s create a sample table and test this approach:
Table creation details
CREATE TABLE filler (
id INT NOT NULL PRIMARY KEY AUTO_INCREMENT
) ENGINE=Memory;
CREATE TABLE logfile (
id INT NOT NULL PRIMARY KEY,
sparse INT NOT NULL,
dense INT NOT NULL,
ts DATETIME NOT NULL,
stuffing VARCHAR(100) NOT NULL,
KEY ix_logfile_ts_id (ts, id),
KEY ix_logfile_sparse_ts_id (sparse, ts, id),
KEY ix_logfile_dense_ts_id (dense, ts, id)
) ENGINE=InnoDB;
DELIMITER $$
CREATE PROCEDURE prc_filler(cnt INT)
BEGIN
DECLARE _cnt INT;
SET _cnt = 1;
WHILE _cnt <= cnt DO
INSERT
INTO filler
SELECT _cnt;
SET _cnt = _cnt + 1;
END WHILE;
END
$$
DELIMITER ;
START TRANSACTION;
CALL prc_filler(500000);
COMMIT;
INSERT
INTO logfile
SELECT id,
CEILING(RAND(20100824) * 30),
CEILING(RAND(20100824 << 1) * 30000),
'2010-08-24' - INTERVAL RAND(20100824 << 2) * 10000000 SECOND,
LPAD('', 100, '*')
FROM filler;
This table has 1,000,000 records.
Instead of a single field, person, I created two different fields: sparse and dense. The first one has 30 distinct values, while the second one has 30,000. This will help us to see how data distribution affects performance of different queries.
Let’s run our original queries. We’ll adjust them a little to help MySQL to pick correct plans:
SELECT *
FROM (
SELECT sparse, MAX(ts) AS last_update
FROM logfile
GROUP BY
sparse
) q
ORDER BY
last_update DESC
LIMIT 20;
View query results
sparse
last_update
15
2010-08-23 23:59:58
26
2010-08-23 23:59:56
11
2010-08-23 23:59:56
30
2010-08-23 23:59:42
29
2010-08-23 23:59:32
13
2010-08-23 23:58:54
27
2010-08-23 23:58:53
7
2010-08-23 23:58:46
5
2010-08-23 23:58:00
12
2010-08-23 23:57:44
14
2010-08-23 23:57:24
6
2010-08-23 23:56:58
2
2010-08-23 23:56:48
24
2010-08-23 23:56:13
17
2010-08-23 23:56:12
23
2010-08-23 23:55:08
19
2010-08-23 23:55:07
20
2010-08-23 23:53:44
10
2010-08-23 23:51:52
4
2010-08-23 23:50:53
20 rows fetched in 0.0005s (0.0026s)
id
select_type
table
type
possible_keys
key
key_len
ref
rows
filtered
Extra
1
PRIMARY
<derived2>
ALL
30
100.00
Using filesort
2
DERIVED
logfile
range
ix_logfile_sparse_ts_id
4
500133
100.00
Using index for group-by
select `q`.`sparse` AS `sparse`,`q`.`last_update` AS `last_update` from (select `20100824_latest`.`logfile`.`sparse` AS `sparse`,max(`20100824_latest`.`logfile`.`ts`) AS `last_update` from `20100824_latest`.`logfile` group by `20100824_latest`.`logfile`.`sparse`) `q` order by `q`.`last_update` desc limit 20
SELECT *
FROM (
SELECT dense, MAX(ts) AS last_update
FROM logfile
GROUP BY
dense
) q
ORDER BY
last_update DESC
LIMIT 20;
View query results
dense
last_update
25324
2010-08-23 23:59:58
13060
2010-08-23 23:59:56
3268
2010-08-23 23:59:56
2327
2010-08-23 23:59:42
23968
2010-08-23 23:59:32
1622
2010-08-23 23:58:54
29693
2010-08-23 23:58:53
655
2010-08-23 23:58:46
5802
2010-08-23 23:58:07
11843
2010-08-23 23:58:00
18894
2010-08-23 23:57:44
6180
2010-08-23 23:57:26
9398
2010-08-23 23:57:24
18012
2010-08-23 23:56:58
25758
2010-08-23 23:56:49
2379
2010-08-23 23:56:48
821
2010-08-23 23:56:39
4186
2010-08-23 23:56:13
20198
2010-08-23 23:56:12
18615
2010-08-23 23:56:01
20 rows fetched in 0.0005s (0.5000s)
id
select_type
table
type
possible_keys
key
key_len
ref
rows
filtered
Extra
1
PRIMARY
<derived2>
ALL
30000
100.00
Using filesort
2
DERIVED
logfile
range
ix_logfile_dense_ts_id
4
500133
100.00
Using index for group-by
select `q`.`dense` AS `dense`,`q`.`last_update` AS `last_update` from (select `20100824_latest`.`logfile`.`dense` AS `dense`,max(`20100824_latest`.`logfile`.`ts`) AS `last_update` from `20100824_latest`.`logfile` group by `20100824_latest`.`logfile`.`dense`) `q` order by `q`.`last_update` desc limit 20
We see that both queries use the same plan and return 20 records, but the first one is instant, while the second one runs for 500 ms. Both queries use filesort, but in second case it has to sort 30,000 records (compared to 30 in the first case).
In this case, it is better to use another approach.
With our original query, we take each person and see which record is latest for this person. But we can as well do it the other way round: take the records in descending order, one by one, and for each record see if it’s latest for this person. If it is, we should return it; if it’s not, this means that the record for this person has already been returned (remember, we take them in descending order).
It’s easy to see that 20 records returned this way will, first, belong to 20 different people, and, second, be the latest records of their respective persons. This is exactly what we need.
The records can easily be scanned in the descending order using the index on (ts, id). But how do we check if the record is the latest? It’s simple: we just take the last record for the given person from the index on (person, ts, id) and compare its id. It takes but a single index seek per record and is almost instant.
Here’s the query to do it:
SELECT id, sparse, dense, ts
FROM logfile lf
WHERE id =
(
SELECT id
FROM logfile lfi
WHERE lfi.sparse = lf.sparse
ORDER BY
sparse DESC, ts DESC, id DESC
LIMIT 1
)
ORDER BY
ts DESC, id DESC
LIMIT 20
id
sparse
dense
ts
121946
15
25324
2010-08-23 23:59:58
276499
11
3268
2010-08-23 23:59:56
62419
26
13060
2010-08-23 23:59:56
254750
30
2327
2010-08-23 23:59:42
96079
29
23968
2010-08-23 23:59:32
290657
13
1622
2010-08-23 23:58:54
278842
27
29693
2010-08-23 23:58:53
329318
7
655
2010-08-23 23:58:46
384956
5
11843
2010-08-23 23:58:00
386333
12
18894
2010-08-23 23:57:44
260404
14
9398
2010-08-23 23:57:24
471000
6
18012
2010-08-23 23:56:58
172079
2
2379
2010-08-23 23:56:48
112653
24
4186
2010-08-23 23:56:13
291683
17
20198
2010-08-23 23:56:12
144673
23
25055
2010-08-23 23:55:08
172118
19
29039
2010-08-23 23:55:07
198913
20
9887
2010-08-23 23:53:44
491436
10
17752
2010-08-23 23:51:52
346651
4
10951
2010-08-23 23:50:53
20 rows fetched in 0.0007s (0.0034s)
id
select_type
table
type
possible_keys
key
key_len
ref
rows
filtered
Extra
1
PRIMARY
lf
index
ix_logfile_ts_id
12
20
2500660.00
Using where
2
DEPENDENT SUBQUERY
lfi
ref
ix_logfile_sparse_ts_id
ix_logfile_sparse_ts_id
4
20100824_latest.lf.sparse
27785
100.00
Using where; Using index
Field or reference '20100824_latest.lf.sparse' of SELECT #2 was resolved in SELECT #1
select `20100824_latest`.`lf`.`id` AS `id`,`20100824_latest`.`lf`.`sparse` AS `sparse`,`20100824_latest`.`lf`.`dense` AS `dense`,`20100824_latest`.`lf`.`ts` AS `ts` from `20100824_latest`.`logfile` `lf` where (`20100824_latest`.`lf`.`id` = (select `20100824_latest`.`lfi`.`id` from `20100824_latest`.`logfile` `lfi` where (`20100824_latest`.`lfi`.`sparse` = `20100824_latest`.`lf`.`sparse`) order by `20100824_latest`.`lfi`.`sparse` desc,`20100824_latest`.`lfi`.`ts` desc,`20100824_latest`.`lfi`.`id` desc limit 1)) order by `20100824_latest`.`lf`.`ts` desc,`20100824_latest`.`lf`.`id` desc limit 20
As we can see, this query uses two different indexes. The first one on (ts, id) is used to scan all records according to the overall timeline; the second one, on (sparse, ts, id) is used to find the id of the latest entry for a person and check if it’s the same as the record selected from the general timeline.
The query is instant: 3 ms.
Let’s check the same query on a column with lots of values:
SELECT id, sparse, dense, ts
FROM logfile lf
WHERE id =
(
SELECT id
FROM logfile lfi
WHERE lfi.dense = lf.dense
ORDER BY
dense DESC, ts DESC, id DESC
LIMIT 1
)
ORDER BY
ts DESC, id DESC
LIMIT 20
id
sparse
dense
ts
121946
15
25324
2010-08-23 23:59:58
276499
11
3268
2010-08-23 23:59:56
62419
26
13060
2010-08-23 23:59:56
254750
30
2327
2010-08-23 23:59:42
96079
29
23968
2010-08-23 23:59:32
290657
13
1622
2010-08-23 23:58:54
278842
27
29693
2010-08-23 23:58:53
329318
7
655
2010-08-23 23:58:46
277612
15
5802
2010-08-23 23:58:07
384956
5
11843
2010-08-23 23:58:00
386333
12
18894
2010-08-23 23:57:44
201899
7
6180
2010-08-23 23:57:26
260404
14
9398
2010-08-23 23:57:24
471000
6
18012
2010-08-23 23:56:58
451808
26
25758
2010-08-23 23:56:49
172079
2
2379
2010-08-23 23:56:48
367042
11
821
2010-08-23 23:56:39
112653
24
4186
2010-08-23 23:56:13
291683
17
20198
2010-08-23 23:56:12
127839
11
18615
2010-08-23 23:56:01
20 rows fetched in 0.0007s (0.0031s)
id
select_type
table
type
possible_keys
key
key_len
ref
rows
filtered
Extra
1
PRIMARY
lf
index
ix_logfile_ts_id
12
20
2500660.00
Using where
2
DEPENDENT SUBQUERY
lfi
ref
ix_logfile_dense_ts_id
ix_logfile_dense_ts_id
4
20100824_latest.lf.dense
8
100.00
Using where; Using index
Field or reference '20100824_latest.lf.dense' of SELECT #2 was resolved in SELECT #1
select `20100824_latest`.`lf`.`id` AS `id`,`20100824_latest`.`lf`.`sparse` AS `sparse`,`20100824_latest`.`lf`.`dense` AS `dense`,`20100824_latest`.`lf`.`ts` AS `ts` from `20100824_latest`.`logfile` `lf` where (`20100824_latest`.`lf`.`id` = (select `20100824_latest`.`lfi`.`id` from `20100824_latest`.`logfile` `lfi` where (`20100824_latest`.`lfi`.`dense` = `20100824_latest`.`lf`.`dense`) order by `20100824_latest`.`lfi`.`dense` desc,`20100824_latest`.`lfi`.`ts` desc,`20100824_latest`.`lfi`.`id` desc limit 1)) order by `20100824_latest`.`lf`.`ts` desc,`20100824_latest`.`lf`.`id` desc limit 20
We see that the query is instant again, despite the data distribution being completely different. This is because the query only skips the records which are not the latest of their persons, and the total number of the records to scan is defined by how many records do we browse before we encounter the 20th unique value in our scan. This value decreases exponentially as the number of distinct persons in the table grows, but with 99% probability it won’t exceed 100 records even for only 20 distinct persons in the table.
The only problem that can arise here is that the number of distinct persons in the table is less than the LIMIT we set. In this case, no new records after the limit is reached can be returned, and a full index scan (accompanied by an index seek once per record) will ultimately be performed.
To work around this, the following simple query can be run in advance:
SELECT COUNT(*)
FROM (
SELECT DISTINCT sparse
FROM logfile
LIMIT 20
) q
COUNT(*)
20
1 row fetched in 0.0001s (0.0015s)
View query details
id
select_type
table
type
possible_keys
key
key_len
ref
rows
filtered
Extra
1
PRIMARY
Select tables optimized away
2
DERIVED
logfile
range
ix_logfile_sparse_ts_id
4
19
100.00
Using index for group-by; Using temporary
select count(0) AS `COUNT(*)` from (select distinct `20100824_latest`.`logfile`.`sparse` AS `sparse` from `20100824_latest`.`logfile` limit 20) `q`
This query will return the actual number of distinct persons in the table if there are less than 20 (or 20 if these are more).
This query is instant even for the dense data:
SELECT COUNT(*)
FROM (
SELECT DISTINCT dense
FROM logfile
LIMIT 20
) q
COUNT(*)
20
1 row fetched in 0.0001s (0.0024s)
View query details
id
select_type
table
type
possible_keys
key
key_len
ref
rows
filtered
Extra
1
PRIMARY
Select tables optimized away
2
DERIVED
logfile
index
ix_logfile_dense_ts_id
16
500132
12.50
Using index; Using temporary
select count(0) AS `COUNT(*)` from (select distinct `20100824_latest`.`logfile`.`dense` AS `dense` from `20100824_latest`.`logfile` limit 20) `q`
This needs to be run as a separate query because MySQL does not allow using anything other than constants in the LIMIT clause. The result of this query should be substituted into the LIMIT clause on the client or in a dynamically composed query on the server.
Summary
To select a number of latest unique records from a table, one can use aggregate functions, however, this can decrease the query performance.
This can be done more efficiently by creating two different indexes on the table and checking the records taken from the general timeline against the end of the index on the person’s timeline.
To avoid performance degradation in marginal cases (when the total number of persons in the table is less than LIMIT), it is possible to make an additional check for the total number of distinct records and adjust the LIMIT clause if there are not enough records.
P. S. I decided to enable comments for the technical posts as well. You are welcome to comment.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 19:00:38 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:21800:"From Stack Overflow:
I have a logfile which logs the insert/delete/updates from all kinds of tables.
I would like to get an overview of for example the last 20 people which records where updated, ordered by the last update (datetime DESC
)
A common solution for such a task would be writing an aggregate query with ORDER BY
and LIMIT
:
SELECT person, MAX(ts) AS last_update
FROM logfile
GROUP BY
person
ORDER BY
last_update DESC
LIMIT 20
What’s bad in this solution? Performance, as usual.
Since last_update
is an aggregate, it cannot be indexed. And ORDER BY
on unindexed fields results in our good old friend, filesort
.
Note that even in this case the indexes can be used and the full table scan can be avoided: if there is an index on (person, ts)
, MySQL
will tend to use a loose index scan on this index, which can save this query if there are relatively few persons in the table. However, if there are many (which is what we can expect for a log table), loose index scan can even degrade performance and generally will be avoided by MySQL
.
We should use another approach here. Let’s create a sample table and test this approach:
Table creation details
CREATE TABLE filler (
id INT NOT NULL PRIMARY KEY AUTO_INCREMENT
) ENGINE=Memory;
CREATE TABLE logfile (
id INT NOT NULL PRIMARY KEY,
sparse INT NOT NULL,
dense INT NOT NULL,
ts DATETIME NOT NULL,
stuffing VARCHAR(100) NOT NULL,
KEY ix_logfile_ts_id (ts, id),
KEY ix_logfile_sparse_ts_id (sparse, ts, id),
KEY ix_logfile_dense_ts_id (dense, ts, id)
) ENGINE=InnoDB;
DELIMITER $$
CREATE PROCEDURE prc_filler(cnt INT)
BEGIN
DECLARE _cnt INT;
SET _cnt = 1;
WHILE _cnt <= cnt DO
INSERT
INTO filler
SELECT _cnt;
SET _cnt = _cnt + 1;
END WHILE;
END
$$
DELIMITER ;
START TRANSACTION;
CALL prc_filler(500000);
COMMIT;
INSERT
INTO logfile
SELECT id,
CEILING(RAND(20100824) * 30),
CEILING(RAND(20100824 << 1) * 30000),
'2010-08-24' - INTERVAL RAND(20100824 << 2) * 10000000 SECOND,
LPAD('', 100, '*')
FROM filler;
This table has 1,000,000 records.
Instead of a single field, person
, I created two different fields: sparse
and dense
. The first one has 30 distinct values, while the second one has 30,000. This will help us to see how data distribution affects performance of different queries.
Let’s run our original queries. We’ll adjust them a little to help MySQL
to pick correct plans:
SELECT *
FROM (
SELECT sparse, MAX(ts) AS last_update
FROM logfile
GROUP BY
sparse
) q
ORDER BY
last_update DESC
LIMIT 20;
View query results
sparse |
last_update |
15 |
2010-08-23 23:59:58 |
26 |
2010-08-23 23:59:56 |
11 |
2010-08-23 23:59:56 |
30 |
2010-08-23 23:59:42 |
29 |
2010-08-23 23:59:32 |
13 |
2010-08-23 23:58:54 |
27 |
2010-08-23 23:58:53 |
7 |
2010-08-23 23:58:46 |
5 |
2010-08-23 23:58:00 |
12 |
2010-08-23 23:57:44 |
14 |
2010-08-23 23:57:24 |
6 |
2010-08-23 23:56:58 |
2 |
2010-08-23 23:56:48 |
24 |
2010-08-23 23:56:13 |
17 |
2010-08-23 23:56:12 |
23 |
2010-08-23 23:55:08 |
19 |
2010-08-23 23:55:07 |
20 |
2010-08-23 23:53:44 |
10 |
2010-08-23 23:51:52 |
4 |
2010-08-23 23:50:53 |
20 rows fetched in 0.0005s (0.0026s) |
id |
select_type |
table |
type |
possible_keys |
key |
key_len |
ref |
rows |
filtered |
Extra |
1 |
PRIMARY |
<derived2> |
ALL |
|
|
|
|
30 |
100.00 |
Using filesort |
2 |
DERIVED |
logfile |
range |
|
ix_logfile_sparse_ts_id |
4 |
|
500133 |
100.00 |
Using index for group-by |
select `q`.`sparse` AS `sparse`,`q`.`last_update` AS `last_update` from (select `20100824_latest`.`logfile`.`sparse` AS `sparse`,max(`20100824_latest`.`logfile`.`ts`) AS `last_update` from `20100824_latest`.`logfile` group by `20100824_latest`.`logfile`.`sparse`) `q` order by `q`.`last_update` desc limit 20
SELECT *
FROM (
SELECT dense, MAX(ts) AS last_update
FROM logfile
GROUP BY
dense
) q
ORDER BY
last_update DESC
LIMIT 20;
View query results
dense |
last_update |
25324 |
2010-08-23 23:59:58 |
13060 |
2010-08-23 23:59:56 |
3268 |
2010-08-23 23:59:56 |
2327 |
2010-08-23 23:59:42 |
23968 |
2010-08-23 23:59:32 |
1622 |
2010-08-23 23:58:54 |
29693 |
2010-08-23 23:58:53 |
655 |
2010-08-23 23:58:46 |
5802 |
2010-08-23 23:58:07 |
11843 |
2010-08-23 23:58:00 |
18894 |
2010-08-23 23:57:44 |
6180 |
2010-08-23 23:57:26 |
9398 |
2010-08-23 23:57:24 |
18012 |
2010-08-23 23:56:58 |
25758 |
2010-08-23 23:56:49 |
2379 |
2010-08-23 23:56:48 |
821 |
2010-08-23 23:56:39 |
4186 |
2010-08-23 23:56:13 |
20198 |
2010-08-23 23:56:12 |
18615 |
2010-08-23 23:56:01 |
20 rows fetched in 0.0005s (0.5000s) |
id |
select_type |
table |
type |
possible_keys |
key |
key_len |
ref |
rows |
filtered |
Extra |
1 |
PRIMARY |
<derived2> |
ALL |
|
|
|
|
30000 |
100.00 |
Using filesort |
2 |
DERIVED |
logfile |
range |
|
ix_logfile_dense_ts_id |
4 |
|
500133 |
100.00 |
Using index for group-by |
select `q`.`dense` AS `dense`,`q`.`last_update` AS `last_update` from (select `20100824_latest`.`logfile`.`dense` AS `dense`,max(`20100824_latest`.`logfile`.`ts`) AS `last_update` from `20100824_latest`.`logfile` group by `20100824_latest`.`logfile`.`dense`) `q` order by `q`.`last_update` desc limit 20
We see that both queries use the same plan and return 20 records, but the first one is instant, while the second one runs for 500 ms. Both queries use filesort, but in second case it has to sort 30,000 records (compared to 30 in the first case).
In this case, it is better to use another approach.
With our original query, we take each person and see which record is latest for this person. But we can as well do it the other way round: take the records in descending order, one by one, and for each record see if it’s latest for this person. If it is, we should return it; if it’s not, this means that the record for this person has already been returned (remember, we take them in descending order).
It’s easy to see that 20 records returned this way will, first, belong to 20 different people, and, second, be the latest records of their respective persons. This is exactly what we need.
The records can easily be scanned in the descending order using the index on (ts, id)
. But how do we check if the record is the latest? It’s simple: we just take the last record for the given person from the index on (person, ts, id)
and compare its id
. It takes but a single index seek per record and is almost instant.
Here’s the query to do it:
SELECT id, sparse, dense, ts
FROM logfile lf
WHERE id =
(
SELECT id
FROM logfile lfi
WHERE lfi.sparse = lf.sparse
ORDER BY
sparse DESC, ts DESC, id DESC
LIMIT 1
)
ORDER BY
ts DESC, id DESC
LIMIT 20
id |
sparse |
dense |
ts |
121946 |
15 |
25324 |
2010-08-23 23:59:58 |
276499 |
11 |
3268 |
2010-08-23 23:59:56 |
62419 |
26 |
13060 |
2010-08-23 23:59:56 |
254750 |
30 |
2327 |
2010-08-23 23:59:42 |
96079 |
29 |
23968 |
2010-08-23 23:59:32 |
290657 |
13 |
1622 |
2010-08-23 23:58:54 |
278842 |
27 |
29693 |
2010-08-23 23:58:53 |
329318 |
7 |
655 |
2010-08-23 23:58:46 |
384956 |
5 |
11843 |
2010-08-23 23:58:00 |
386333 |
12 |
18894 |
2010-08-23 23:57:44 |
260404 |
14 |
9398 |
2010-08-23 23:57:24 |
471000 |
6 |
18012 |
2010-08-23 23:56:58 |
172079 |
2 |
2379 |
2010-08-23 23:56:48 |
112653 |
24 |
4186 |
2010-08-23 23:56:13 |
291683 |
17 |
20198 |
2010-08-23 23:56:12 |
144673 |
23 |
25055 |
2010-08-23 23:55:08 |
172118 |
19 |
29039 |
2010-08-23 23:55:07 |
198913 |
20 |
9887 |
2010-08-23 23:53:44 |
491436 |
10 |
17752 |
2010-08-23 23:51:52 |
346651 |
4 |
10951 |
2010-08-23 23:50:53 |
20 rows fetched in 0.0007s (0.0034s) |
id |
select_type |
table |
type |
possible_keys |
key |
key_len |
ref |
rows |
filtered |
Extra |
1 |
PRIMARY |
lf |
index |
|
ix_logfile_ts_id |
12 |
|
20 |
2500660.00 |
Using where |
2 |
DEPENDENT SUBQUERY |
lfi |
ref |
ix_logfile_sparse_ts_id |
ix_logfile_sparse_ts_id |
4 |
20100824_latest.lf.sparse |
27785 |
100.00 |
Using where; Using index |
Field or reference '20100824_latest.lf.sparse' of SELECT #2 was resolved in SELECT #1
select `20100824_latest`.`lf`.`id` AS `id`,`20100824_latest`.`lf`.`sparse` AS `sparse`,`20100824_latest`.`lf`.`dense` AS `dense`,`20100824_latest`.`lf`.`ts` AS `ts` from `20100824_latest`.`logfile` `lf` where (`20100824_latest`.`lf`.`id` = (select `20100824_latest`.`lfi`.`id` from `20100824_latest`.`logfile` `lfi` where (`20100824_latest`.`lfi`.`sparse` = `20100824_latest`.`lf`.`sparse`) order by `20100824_latest`.`lfi`.`sparse` desc,`20100824_latest`.`lfi`.`ts` desc,`20100824_latest`.`lfi`.`id` desc limit 1)) order by `20100824_latest`.`lf`.`ts` desc,`20100824_latest`.`lf`.`id` desc limit 20
As we can see, this query uses two different indexes. The first one on (ts, id)
is used to scan all records according to the overall timeline; the second one, on (sparse, ts, id)
is used to find the id
of the latest entry for a person and check if it’s the same as the record selected from the general timeline.
The query is instant: 3 ms.
Let’s check the same query on a column with lots of values:
SELECT id, sparse, dense, ts
FROM logfile lf
WHERE id =
(
SELECT id
FROM logfile lfi
WHERE lfi.dense = lf.dense
ORDER BY
dense DESC, ts DESC, id DESC
LIMIT 1
)
ORDER BY
ts DESC, id DESC
LIMIT 20
id |
sparse |
dense |
ts |
121946 |
15 |
25324 |
2010-08-23 23:59:58 |
276499 |
11 |
3268 |
2010-08-23 23:59:56 |
62419 |
26 |
13060 |
2010-08-23 23:59:56 |
254750 |
30 |
2327 |
2010-08-23 23:59:42 |
96079 |
29 |
23968 |
2010-08-23 23:59:32 |
290657 |
13 |
1622 |
2010-08-23 23:58:54 |
278842 |
27 |
29693 |
2010-08-23 23:58:53 |
329318 |
7 |
655 |
2010-08-23 23:58:46 |
277612 |
15 |
5802 |
2010-08-23 23:58:07 |
384956 |
5 |
11843 |
2010-08-23 23:58:00 |
386333 |
12 |
18894 |
2010-08-23 23:57:44 |
201899 |
7 |
6180 |
2010-08-23 23:57:26 |
260404 |
14 |
9398 |
2010-08-23 23:57:24 |
471000 |
6 |
18012 |
2010-08-23 23:56:58 |
451808 |
26 |
25758 |
2010-08-23 23:56:49 |
172079 |
2 |
2379 |
2010-08-23 23:56:48 |
367042 |
11 |
821 |
2010-08-23 23:56:39 |
112653 |
24 |
4186 |
2010-08-23 23:56:13 |
291683 |
17 |
20198 |
2010-08-23 23:56:12 |
127839 |
11 |
18615 |
2010-08-23 23:56:01 |
20 rows fetched in 0.0007s (0.0031s) |
id |
select_type |
table |
type |
possible_keys |
key |
key_len |
ref |
rows |
filtered |
Extra |
1 |
PRIMARY |
lf |
index |
|
ix_logfile_ts_id |
12 |
|
20 |
2500660.00 |
Using where |
2 |
DEPENDENT SUBQUERY |
lfi |
ref |
ix_logfile_dense_ts_id |
ix_logfile_dense_ts_id |
4 |
20100824_latest.lf.dense |
8 |
100.00 |
Using where; Using index |
Field or reference '20100824_latest.lf.dense' of SELECT #2 was resolved in SELECT #1
select `20100824_latest`.`lf`.`id` AS `id`,`20100824_latest`.`lf`.`sparse` AS `sparse`,`20100824_latest`.`lf`.`dense` AS `dense`,`20100824_latest`.`lf`.`ts` AS `ts` from `20100824_latest`.`logfile` `lf` where (`20100824_latest`.`lf`.`id` = (select `20100824_latest`.`lfi`.`id` from `20100824_latest`.`logfile` `lfi` where (`20100824_latest`.`lfi`.`dense` = `20100824_latest`.`lf`.`dense`) order by `20100824_latest`.`lfi`.`dense` desc,`20100824_latest`.`lfi`.`ts` desc,`20100824_latest`.`lfi`.`id` desc limit 1)) order by `20100824_latest`.`lf`.`ts` desc,`20100824_latest`.`lf`.`id` desc limit 20
We see that the query is instant again, despite the data distribution being completely different. This is because the query only skips the records which are not the latest of their persons, and the total number of the records to scan is defined by how many records do we browse before we encounter the 20th unique value in our scan. This value decreases exponentially as the number of distinct persons in the table grows, but with 99% probability it won’t exceed 100 records even for only 20 distinct persons in the table.
The only problem that can arise here is that the number of distinct persons in the table is less than the LIMIT
we set. In this case, no new records after the limit is reached can be returned, and a full index scan (accompanied by an index seek once per record) will ultimately be performed.
To work around this, the following simple query can be run in advance:
SELECT COUNT(*)
FROM (
SELECT DISTINCT sparse
FROM logfile
LIMIT 20
) q
COUNT(*) |
20 |
1 row fetched in 0.0001s (0.0015s) |
View query details
id |
select_type |
table |
type |
possible_keys |
key |
key_len |
ref |
rows |
filtered |
Extra |
1 |
PRIMARY |
|
|
|
|
|
|
|
|
Select tables optimized away |
2 |
DERIVED |
logfile |
range |
|
ix_logfile_sparse_ts_id |
4 |
|
19 |
100.00 |
Using index for group-by; Using temporary |
select count(0) AS `COUNT(*)` from (select distinct `20100824_latest`.`logfile`.`sparse` AS `sparse` from `20100824_latest`.`logfile` limit 20) `q`
This query will return the actual number of distinct persons in the table if there are less than 20 (or 20 if these are more).
This query is instant even for the dense data:
SELECT COUNT(*)
FROM (
SELECT DISTINCT dense
FROM logfile
LIMIT 20
) q
COUNT(*) |
20 |
1 row fetched in 0.0001s (0.0024s) |
View query details
id |
select_type |
table |
type |
possible_keys |
key |
key_len |
ref |
rows |
filtered |
Extra |
1 |
PRIMARY |
|
|
|
|
|
|
|
|
Select tables optimized away |
2 |
DERIVED |
logfile |
index |
|
ix_logfile_dense_ts_id |
16 |
|
500132 |
12.50 |
Using index; Using temporary |
select count(0) AS `COUNT(*)` from (select distinct `20100824_latest`.`logfile`.`dense` AS `dense` from `20100824_latest`.`logfile` limit 20) `q`
This needs to be run as a separate query because MySQL does not allow using anything other than constants in the LIMIT
clause. The result of this query should be substituted into the LIMIT
clause on the client or in a dynamically composed query on the server.
Summary
To select a number of latest unique records from a table, one can use aggregate functions, however, this can decrease the query performance.
This can be done more efficiently by creating two different indexes on the table and checking the records taken from the general timeline against the end of the index on the person’s timeline.
To avoid performance degradation in marginal cases (when the total number of persons in the table is less than LIMIT
), it is possible to make an additional check for the total number of distinct records and adjust the LIMIT
clause if there are not enough records.
P. S. I decided to enable comments for the technical posts as well. You are welcome to comment.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:23:"Alex Bolenok (Quassnoi)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:17;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:32:"Using Pentaho Spoon to load data";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:49:"http://opensourcedba.wordpress.com/2010/08/24/25/";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:49:"http://opensourcedba.wordpress.com/2010/08/24/25/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:4528:"Pentaho’s Spoon is an open source ETL or Extract, Transform and Load tool that makes loading data from various formats into a MySQL server easy. Spoon also lets you check the data for problems and correct them before it gets inserted. It does a lot of things very well to make life easier for a DBA. So if people are entering their state of residence as ‘CA’, ‘Cal’, ‘Cal.’ and ‘California’ , Spoon can clean up the data to what you need.
What follows is an example of transforming a CSV file into a series on INSERT statements. It is a very simple transformation but is a good initial exposure to how Spoon functions. It then covers how to use Spoon with a bulk loading program.
There have been questions on the Calpont InfiniDB forums (http://www.infinifb.org/forums) on using Spoon to drive the cpimport bulk loader program. LOAD DATA INFILE can be slow (10,000 records a second versus 1.5 million a second for cpimport). And for large data sets used with Calpont’s InfiniDB storage engine, Spoon can be used to set up a job that becomes a ‘one button’ to push data import or scheduled to be run at a predetermined time.
The is the Welcome screen for spoon
File -> New -> Transformation
The big canvas in the middle is for the layout of the transformation
From the Steps column on the left, expand the Output folder and drag CSV File Input to the canvas
A simple two column comma separated file was created and the location is input to the Filename line
1,100
2,200
3,300
4,400
5,500
6,601
7,702
8,803
9,904
The contents of the CSV file
CREATE TABLE x (
x INT, y INT) ENGINE=InfiniDB;
The DDL to create the target table. This table was created in the test database.
Select Preview and note that Spoon is seeing the two desired columns
Now a second Step or hop is needed. From the Output folder under Steps, drag the SQL File Output icon
to the canvas. Hold down the SHIFT key and drag the mouse from the CSV file input icon to the SQL File Output to designate the flow of data between the steps.
Double click on the SQL File Output icon and configure the output from this step. Select New to the right of the Connection line.
The connection is configured for an InfiniDB instance on the local system. Select the Test button
The connection must be OK
Select the green arrow icon to run the transformation. Select Launch on the next screen
Under Execution Results, the transformation can be monitored. Here all 9 lines from the CSV file are processed in the two hops.
INSERT INTO test.x(Field_000, Field_001) VALUES (1,100);INSERT INTO test.x(Field_000, Field_001) VALUES (2,200);INSERT INTO test.x(Field_000, Field_001) VALUES (3,300);INSERT INTO test.x(Field_000, Field_001) VALUES (4,400);INSERT INTO test.x(Field_000, Field_001) VALUES (5,500);INSERT INTO test.x(Field_000, Field_001) VALUES (6,601);INSERT INTO test.x(Field_000, Field_001) VALUES (7,702);INSERT INTO test.x(Field_000, Field_001) VALUES (8,803);INSERT INTO test.x(Field_000, Field_001) VALUES (9,904);
This file can now be sourced from the MySQL command line. However for InfiniDB this is VERY slow for large data sets.
To take advantage of the cpimport bulk loader, create a new transformation and select Text File Output from the Output folder. Hold the shift key and drag the mouse from the CSV File Input icon to the Text file output icon to set the data flow.
Double click the Text file output icon. The contents of the transformed data need to be written to a file in the InfiniDB bulk import directory, /usr/local/Calpont/data/bulk/data/import, and is named x. Be sure to set the Extension to tbl.
Under the Content tab, set the Separator to a pipe character (|).
Under the Field tab, select Get Fields. Then select OK
/usr/local/Calpont/bin/colxml –j 101 –l x.tbl –t x test
The parameters for the bulk import are set with the colxml utility. An unused job number was selected (101). The other arguments declare that data from the file x.tbl will be loaded into the table x in the test database. Enter this command at a shell prompt/
Another hop is established. Use the steps from the previous hop.
Double click on Text file output2. Set Filename to /usr/local/Calpont/bin/cpimport -j 101, click ‘Run this as a script instead?‘, and then click OK
Execute the transformation and the CSV file will be loaded into the InfinDB instance.
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 16:45:32 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:13:"Uncategorized";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:9719:"Pentaho’s Spoon is an open source ETL or Extract, Transform and Load tool that makes loading data from various formats into a MySQL server easy. Spoon also lets you check the data for problems and correct them before it gets inserted. It does a lot of things very well to make life easier for a DBA. So if people are entering their state of residence as ‘CA’, ‘Cal’, ‘Cal.’ and ‘California’ , Spoon can clean up the data to what you need.
What follows is an example of transforming a CSV file into a series on INSERT statements. It is a very simple transformation but is a good initial exposure to how Spoon functions. It then covers how to use Spoon with a bulk loading program.
There have been questions on the Calpont InfiniDB forums (http://www.infinifb.org/forums) on using Spoon to drive the cpimport bulk loader program. LOAD DATA INFILE can be slow (10,000 records a second versus 1.5 million a second for cpimport). And for large data sets used with Calpont’s InfiniDB storage engine, Spoon can be used to set up a job that becomes a ‘one button’ to push data import or scheduled to be run at a predetermined time.
|
The is the Welcome screen for spoon |
|
File -> New -> Transformation |
|
The big canvas in the middle is for the layout of the transformation |
|
From the Steps column on the left, expand the Output folder and drag CSV File Input to the canvas |
|
A simple two column comma separated file was created and the location is input to the Filename line |
1,100
2,200
3,300
4,400
5,500
6,601
7,702
8,803
9,904
|
The contents of the CSV file |
CREATE TABLE x (
x INT, y INT) ENGINE=InfiniDB; |
The DDL to create the target table. This table was created in the test database. |
|
Select Preview and note that Spoon is seeing the two desired columns |
|
Now a second Step or hop is needed. From the Output folder under Steps, drag the SQL File Output icon
to the canvas. Hold down the SHIFT key and drag the mouse from the CSV file input icon to the SQL File Output to designate the flow of data between the steps. |
|
Double click on the SQL File Output icon and configure the output from this step. Select New to the right of the Connection line. |
|
The connection is configured for an InfiniDB instance on the local system. Select the Test button |
|
The connection must be OK |
|
Select the green arrow icon to run the transformation. Select Launch on the next screen |
|
Under Execution Results, the transformation can be monitored. Here all 9 lines from the CSV file are processed in the two hops. |
INSERT INTO test.x(Field_000, Field_001) VALUES (1,100);INSERT INTO test.x(Field_000, Field_001) VALUES (2,200);INSERT INTO test.x(Field_000, Field_001) VALUES (3,300);INSERT INTO test.x(Field_000, Field_001) VALUES (4,400);INSERT INTO test.x(Field_000, Field_001) VALUES (5,500);INSERT INTO test.x(Field_000, Field_001) VALUES (6,601);INSERT INTO test.x(Field_000, Field_001) VALUES (7,702);INSERT INTO test.x(Field_000, Field_001) VALUES (8,803);INSERT INTO test.x(Field_000, Field_001) VALUES (9,904);
|
This file can now be sourced from the MySQL command line. However for InfiniDB this is VERY slow for large data sets. |
|
To take advantage of the cpimport bulk loader, create a new transformation and select Text File Output from the Output folder. Hold the shift key and drag the mouse from the CSV File Input icon to the Text file output icon to set the data flow. |
|
Double click the Text file output icon. The contents of the transformed data need to be written to a file in the InfiniDB bulk import directory, /usr/local/Calpont/data/bulk/data/import, and is named x. Be sure to set the Extension to tbl. |
|
Under the Content tab, set the Separator to a pipe character (|). |
|
Under the Field tab, select Get Fields. Then select OK |
/usr/local/Calpont/bin/colxml –j 101 –l x.tbl –t x test |
The parameters for the bulk import are set with the colxml utility. An unused job number was selected (101). The other arguments declare that data from the file x.tbl will be loaded into the table x in the test database. Enter this command at a shell prompt/ |
|
Another hop is established. Use the steps from the previous hop. |
|
Double click on Text file output2. Set Filename to /usr/local/Calpont/bin/cpimport -j 101, click ‘Run this as a script instead?‘, and then click OK |
|
Execute the transformation and the CSV file will be loaded into the InfinDB instance. |
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:11:"Dave Stokes";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:18;a:6:{s:4:"data";s:153:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:25:"451 CAOS Links 2010.08.24";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:47:"http://blogs.the451group.com/opensource/?p=2225";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:60:"http://feedproxy.google.com/~r/451opensource/~3/8SesvGXcoBo/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1829:"The future of open source licensing. OpenSolaris governing board quits. And more.
Follow 451 CAOS Links live @caostheory on Twitter and Identi.ca
“Tracking the open source news wires, so you don’t have to.”
# Glyn Moody asked which open source software licensing is best for the future?
# The OpenSolaris Governing Board has collectively and expectedly resigned.
# OpenBravo has updated its rapid implementation ERP offering for small and mid-sized businesses, Openbravo QuickStart.
# Red Hat has launched Extended Life Cycle Support (ELS) for Red Hat Enterprise Linux.
# GigaOm published a video interview with Marten Mickos on open source and cloud computing.
# The UK government made its first contributions to Drupal.
# As Matt Asay noted, open core naysayers appear to be promoting proprietary software.
# Cloud.com’s CloudStack open source private and public cloud computing software now supports VMware.
# Gluster introduced VMStor, which simplifies scalable NAS for virtual machine storage.
# Microsoft expressed its love for open source.
# newScale, rPath and Eucalyptus Systems have introduced an integrated platform for enterprise private and hybrid cloud computing.
# Fortune magazine examined how corporate America went open source.
# Martin Michlmayr continued his examination of open source contributor agreements with some examples.
# DotNetNuke introduced DotNetNuke Enterprise Edition, including the new DotNetNuke Content Staging feature.
# Brian Proffitt tackled criticism of Linux Foundation and Open Invention Network, in the light of Oracle vs Google.
# Palamida joined the Linux Foundation.
# OpenStack published an update on its progress.
# Black Duck analysis shows 70% growth from 2008 to 2009 in open source projects specifically associated with cloud computing.
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 16:31:34 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:23:{i:0;a:5:{s:4:"data";s:8:"Software";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:9:"451 group";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:13:"451caostheory";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:8:"451group";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:10:"black duck";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:10:"caostheory";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:9:"cloud.com";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:20:"copyright assignment";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:8;a:5:{s:4:"data";s:10:"DotNetNuke";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:9;a:5:{s:4:"data";s:6:"drupal";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:10;a:5:{s:4:"data";s:10:"Eucalpytus";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:11;a:5:{s:4:"data";s:7:"fortune";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:12;a:5:{s:4:"data";s:7:"Gluster";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:13;a:5:{s:4:"data";s:10:"glyn moody";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:14;a:5:{s:4:"data";s:6:"google";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:15;a:5:{s:4:"data";s:5:"Linux";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:16;a:5:{s:4:"data";s:16:"Linux Foundation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:17;a:5:{s:4:"data";s:13:"marten mickos";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:18;a:5:{s:4:"data";s:11:"matt aslett";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:19;a:5:{s:4:"data";s:10:"mattaslett";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:20;a:5:{s:4:"data";s:14:"matthew aslett";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:21;a:5:{s:4:"data";s:13:"matthewaslett";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:22;a:5:{s:4:"data";s:8:"microsof";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3419:"The future of open source licensing. OpenSolaris governing board quits. And more.
Follow 451 CAOS Links live @caostheory on Twitter and Identi.ca
“Tracking the open source news wires, so you don’t have to.”
# Glyn Moody asked which open source software licensing is best for the future?
# The OpenSolaris Governing Board has collectively and expectedly resigned.
# OpenBravo has updated its rapid implementation ERP offering for small and mid-sized businesses, Openbravo QuickStart.
# Red Hat has launched Extended Life Cycle Support (ELS) for Red Hat Enterprise Linux.
# GigaOm published a video interview with Marten Mickos on open source and cloud computing.
# The UK government made its first contributions to Drupal.
# As Matt Asay noted, open core naysayers appear to be promoting proprietary software.
# Cloud.com’s CloudStack open source private and public cloud computing software now supports VMware.
# Gluster introduced VMStor, which simplifies scalable NAS for virtual machine storage.
# Microsoft expressed its love for open source.
# newScale, rPath and Eucalyptus Systems have introduced an integrated platform for enterprise private and hybrid cloud computing.
# Fortune magazine examined how corporate America went open source.
# Martin Michlmayr continued his examination of open source contributor agreements with some examples.
# DotNetNuke introduced DotNetNuke Enterprise Edition, including the new DotNetNuke Content Staging feature.
# Brian Proffitt tackled criticism of Linux Foundation and Open Invention Network, in the light of Oracle vs Google.
# Palamida joined the Linux Foundation.
# OpenStack published an update on its progress.
# Black Duck analysis shows 70% growth from 2008 to 2009 in open source projects specifically associated with cloud computing.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"The 451 Group";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:19;a:6:{s:4:"data";s:53:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:81:"SQLyog – MySQL GUI 8.6 GA – new features, improved performance and stability.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:34:"http://www.webyog.com/blog/?p=2006";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:120:"http://www.webyog.com/blog/2010/08/24/sqlyog-%E2%80%93-mysql-gui-8-6-ga-new-features-improved-performance-and-stability/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2946:"We are pleased to announce the release of SQLyog – MySQL GUI 8.6 GA. This release adds functionalities requested by users for some time, and fixes a number of bugs. Most important enhancements:
User management has been updated with a brand-new interface, has been completely rewritten and also now fully communicates with the MySQL server the recommended way using GRANT and REVOKE syntax. We believe that with this release we have provided the best available ever GUI for MySQL User Management.
For users that prefer to work in a spreadsheet-like interface when filtering and sorting data the options to do this have been enhanced: There is now a ‘custom filter’ option to be used when filtering on a value that does not exist in the result set displayed – or even is not stored in the table at all. Additionally you may now define the substring to be filtered on in more flexible ways than before.
In the editor we added “parenthesis’es matching”. With complex statements (JOINs on derived tables, SUBQUERIES, statements with nested functions and similar) this will make it much easier to identify the structure of the statement. Just position the cursor after a parenthesis and the actual parenthesis and its match will highlight.
We have improved performance by optimizing code in code segments executed most frequently. Additionally we have deployed more debugging and performance measuring tools. This includes – but is not restricted to – moving our build environment to latest Visual Studio environment (2010 edition).
Note that with this release we have stopped supporting Windows 2000. It has for some time been increasingly difficult to continue this support. It is actually quite hard to find a recent system where Windows 2000 will run without errors and we have over the few last years spent quite a lot of effort fixing issues occurring only on this variant of Windows. We believe that the effort is better used ensuring optimal performance on recent and still supported Windows variants – not at least Windows 7, what fastly is taking the position of the dominating Windows variant. The move to Visual Studio 2010 shall be seen in this context. But of course XP, Vista (and the same generations of Windows server variants: 2003 and 2008) are still supported as well as Wine.
Links to Beta/RC release blogs with detailed information.
* http://www.webyog.com/blog/2010/08/17/sqlyog-mysql-gui-8-6-rc3-released/
* http://www.webyog.com/blog/2010/08/06/sqlyog-mysql-gui-8-6-rc2-released/
* http://www.webyog.com/blog/2010/08/06/sqlyog-mysql-gui-8-6-rc-released/
* http://www.webyog.com/blog/2010/08/03/sqlyog-mysql-gui-8-6-beta3-released/
* http://www.webyog.com/blog/2010/07/22/sqlyog-mysql-gui-8-6-beta2-released/
* http://www.webyog.com/blog/2010/07/20/sqlyog-mysql-gui-8-6-beta1-released/
Downloads: http://webyog.com/en/downloads.php
Purchase: http://webyog.com/en/buy.php";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 14:54:57 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:3:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:8:"Releases";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:6:"SQLyog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:4424:"We are pleased to announce the release of SQLyog – MySQL GUI 8.6 GA. This release adds functionalities requested by users for some time, and fixes a number of bugs. Most important enhancements:
User management has been updated with a brand-new interface, has been completely rewritten and also now fully communicates with the MySQL server the recommended way using GRANT and REVOKE syntax. We believe that with this release we have provided the best available ever GUI for MySQL User Management.
For users that prefer to work in a spreadsheet-like interface when filtering and sorting data the options to do this have been enhanced: There is now a ‘custom filter’ option to be used when filtering on a value that does not exist in the result set displayed – or even is not stored in the table at all. Additionally you may now define the substring to be filtered on in more flexible ways than before.
In the editor we added “parenthesis’es matching”. With complex statements (JOINs on derived tables, SUBQUERIES, statements with nested functions and similar) this will make it much easier to identify the structure of the statement. Just position the cursor after a parenthesis and the actual parenthesis and its match will highlight.
We have improved performance by optimizing code in code segments executed most frequently. Additionally we have deployed more debugging and performance measuring tools. This includes – but is not restricted to – moving our build environment to latest Visual Studio environment (2010 edition).
Note that with this release we have stopped supporting Windows 2000. It has for some time been increasingly difficult to continue this support. It is actually quite hard to find a recent system where Windows 2000 will run without errors and we have over the few last years spent quite a lot of effort fixing issues occurring only on this variant of Windows. We believe that the effort is better used ensuring optimal performance on recent and still supported Windows variants – not at least Windows 7, what fastly is taking the position of the dominating Windows variant. The move to Visual Studio 2010 shall be seen in this context. But of course XP, Vista (and the same generations of Windows server variants: 2003 and 2008) are still supported as well as Wine.
Links to Beta/RC release blogs with detailed information.
* http://www.webyog.com/blog/2010/08/17/sqlyog-mysql-gui-8-6-rc3-released/
* http://www.webyog.com/blog/2010/08/06/sqlyog-mysql-gui-8-6-rc2-released/
* http://www.webyog.com/blog/2010/08/06/sqlyog-mysql-gui-8-6-rc-released/
* http://www.webyog.com/blog/2010/08/03/sqlyog-mysql-gui-8-6-beta3-released/
* http://www.webyog.com/blog/2010/07/22/sqlyog-mysql-gui-8-6-beta2-released/
* http://www.webyog.com/blog/2010/07/20/sqlyog-mysql-gui-8-6-beta1-released/
Downloads: http://webyog.com/en/downloads.php
Purchase: http://webyog.com/en/buy.php
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:6:"Webyog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:20;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:36:"OpenSQLCamp Boston hotel information";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:31:"1077 at http://technocation.org";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:68:"http://technocation.org/content/opensqlcamp-boston-hotel-information";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:993:"I am very happy to announce that I have secured a great rate at a hotel for OpenSQLCamp (a free weekend conference for open source databases such as MySQL, Postgres, SQLite, and NoSQL databases). We have a room block at the Doubletree Club Hotel Boston Bayside at 24 Mt. Vernon Street in Boston at a rate of $149 per night, for single or double occupancy*, for both Friday night, October 15 and Saturday night, October 16. Wireless internet access, which is usually $9.95 per night, is included in the room fee, so there's no hidden extra there. There is also a free shuttle from Boston's Logan Airport to the hotel**. The subway is steps away from both the hotel and the venue (MIT Stata Center on the corner of Main and Vassar Streets in Cambridge). Subway fare is $2.00 each way, so if you stay at the hotel you'd have $8 extra in transportation fee.
You can reserve your room by booking here. You are responsible for paying for your own room. Some more useful information:
read more";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 13:13:55 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:4:"News";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:11:"Conferences";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1425:"I am very happy to announce that I have secured a great rate at a hotel for OpenSQLCamp (a free weekend conference for open source databases such as MySQL, Postgres, SQLite, and NoSQL databases). We have a room block at the Doubletree Club Hotel Boston Bayside at 24 Mt. Vernon Street in Boston at a rate of $149 per night, for single or double occupancy*, for both Friday night, October 15 and Saturday night, October 16. Wireless internet access, which is usually $9.95 per night, is included in the room fee, so there's no hidden extra there. There is also a free shuttle from Boston's Logan Airport to the hotel**. The subway is steps away from both the hotel and the venue (MIT Stata Center on the corner of Main and Vassar Streets in Cambridge). Subway fare is $2.00 each way, so if you stay at the hotel you'd have $8 extra in transportation fee.
You can reserve your room by booking here. You are responsible for paying for your own room. Some more useful information:
read more
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Technocation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:21;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:56:"For the n-th time, ReiserFS is not a cluster file system";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:34:"http://fghaas.wordpress.com/?p=577";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:95:"http://fghaas.wordpress.com/2010/08/24/for-the-n-th-time-reiserfs-is-not-a-cluster-file-system/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:564:"Neither is ext3. Nor ext4. Nor btrfs. And thus, none of these will work on dual-Primary DRBD. Nor active-active shared storage. Nor any synchronously replicated active-active SAN. And we’re telling you very clearly.
So if you choose to ignore all warnings and put ReiserFS on dual-Primary DRBD, and mount it from two nodes, you’ve just signed up for wrecking your data. And when that happens, don’t come whining. And don’t blame DRBD or any other of the technologies you may be choosing to employ while ignoring the documentation.
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 13:08:22 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:4:"Rant";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2511:"Neither is ext3. Nor ext4. Nor btrfs. And thus, none of these will work on dual-Primary DRBD. Nor active-active shared storage. Nor any synchronously replicated active-active SAN. And we’re telling you very clearly.
So if you choose to ignore all warnings and put ReiserFS on dual-Primary DRBD, and mount it from two nodes, you’ve just signed up for wrecking your data. And when that happens, don’t come whining. And don’t blame DRBD or any other of the technologies you may be choosing to employ while ignoring the documentation.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:15:"Florian G. Haas";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:22;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:29:"EMT Tutorial – Installation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:35:"http://ebergen.net/wordpress/?p=393";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:55:"http://ebergen.net/wordpress/2010/08/23/installing-emt/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:3734:"EMT is a monitoring tool that I’ve been developing over the past few years. It’s goal is to serve as a hub for performance metrics on a single server. I’ve tried to talk about what EMT is before but I’m not a very good writer so I thought it would be best to just show people. This tutorial is going to be a quick overview of installing EMT from the rpm and a basic tutorial of it’s usage. Some of this is covered in the manual and some has changed in newer released.
Installation
The easiest way to install EMT is to grab the latest rpm from the Google Code downloads page. After installing the rpm you will see a notice about correcting some details in the default view.
ebergen@etna:(~/gc/emt) sudo rpm -i ./emt-0.2-107.noarch.rpm
The emt_view command will likely having missing data until you specify correct interfaces and disks in /opt/emt/plugins/views/default.php.
Seeing this reminds me that EMT needs to support user defined views. I’ve filed an issue about this. Moving on..
The default system stats plugin for EMT uses some tools that are probably already on your system to collect stats about system performance. This duplicates a lot of functionality from the sysstat package which is fine because EMT isn’t just about system stats. If everything went OK in a few minutes the emt_view command will output some stats.
Basic Usage
ebergen@etna:(~/gc/emt) emt_view -n 5
[-------emt-------] [--cpu--] [disk memory [------network------] [----swap----]
Sampling Start Time Sys% Usr% Busy% [Mem%] Recv Bytes Send Bytes [In] [Out Used
sda eth0 eth0
2010-08-16 20:40:01 0 0 1 42 23K 89K 0 0 700M
2010-08-16 20:42:02 1 1 2 42 33K 110K 0 0 700M
2010-08-16 20:43:02 0 1 0 42 21K 93K 0 0 700M
2010-08-16 20:44:01 0 0 0 42 19K 144K 0 0 700M
2010-08-16 20:45:01 0 1 0 42 29K 159K 0 0 700M
The -n 5 tells emt_view to return the most recent 5 events which in the default configuration is 5 minutes of data. EMT plugins are divided into two parts, commands and fields. Internally every minute a series of commands are executed and one or more fields is parsed from those commands. In the output each column is one field. The power of EMT is being able to compare the results of any command with any other command side by side. In the above output there are the results from at least 5 commands.
Depending on the fields there will be either two or three headings per column. The first is the namespace. In future releases it will prevent name collisions between plugins but for now it’s only real use is a grouping for headings. The second column is the field name. The third column is a sub field. Sub fields can be dynamically discovered each minute by a plugin. In this case the plugin discovered eth0 and the view is configured to use it.
A few different views ship with EMT. Some of these are as simple as a list of fields to display. Others create fields on the fly. To see the list of views use emt_view -v. To select a view use emt_view -s view_name. It’s possible to create custom views on the command line with the fields listed in emt_view -l. I’ll cover this in more detail in a future post.
emt_view is the basic method of accessing the data provided by EMT. There are also other programs such as emt_awk which provide csv output that can be piped to other commands like awk. emt_view is commonly used for analysis and emt_awk is often used by monitoring tools to alert on thresholds. I’ll cover these and other commands in future tutorials.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 01:12:46 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:3:"EMT";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:4309:"EMT is a monitoring tool that I’ve been developing over the past few years. It’s goal is to serve as a hub for performance metrics on a single server. I’ve tried to talk about what EMT is before but I’m not a very good writer so I thought it would be best to just show people. This tutorial is going to be a quick overview of installing EMT from the rpm and a basic tutorial of it’s usage. Some of this is covered in the manual and some has changed in newer released.
Installation
The easiest way to install EMT is to grab the latest rpm from the Google Code downloads page. After installing the rpm you will see a notice about correcting some details in the default view.
ebergen@etna:(~/gc/emt) sudo rpm -i ./emt-0.2-107.noarch.rpm
The emt_view command will likely having missing data until you specify correct interfaces and disks in /opt/emt/plugins/views/default.php.
Seeing this reminds me that EMT needs to support user defined views. I’ve filed an issue about this. Moving on..
The default system stats plugin for EMT uses some tools that are probably already on your system to collect stats about system performance. This duplicates a lot of functionality from the sysstat package which is fine because EMT isn’t just about system stats. If everything went OK in a few minutes the emt_view command will output some stats.
Basic Usage
ebergen@etna:(~/gc/emt) emt_view -n 5
[-------emt-------] [--cpu--] [disk memory [------network------] [----swap----]
Sampling Start Time Sys% Usr% Busy% [Mem%] Recv Bytes Send Bytes [In] [Out Used
sda eth0 eth0
2010-08-16 20:40:01 0 0 1 42 23K 89K 0 0 700M
2010-08-16 20:42:02 1 1 2 42 33K 110K 0 0 700M
2010-08-16 20:43:02 0 1 0 42 21K 93K 0 0 700M
2010-08-16 20:44:01 0 0 0 42 19K 144K 0 0 700M
2010-08-16 20:45:01 0 1 0 42 29K 159K 0 0 700M
The -n 5 tells emt_view to return the most recent 5 events which in the default configuration is 5 minutes of data. EMT plugins are divided into two parts, commands and fields. Internally every minute a series of commands are executed and one or more fields is parsed from those commands. In the output each column is one field. The power of EMT is being able to compare the results of any command with any other command side by side. In the above output there are the results from at least 5 commands.
Depending on the fields there will be either two or three headings per column. The first is the namespace. In future releases it will prevent name collisions between plugins but for now it’s only real use is a grouping for headings. The second column is the field name. The third column is a sub field. Sub fields can be dynamically discovered each minute by a plugin. In this case the plugin discovered eth0 and the view is configured to use it.
A few different views ship with EMT. Some of these are as simple as a list of fields to display. Others create fields on the fly. To see the list of views use emt_view -v. To select a view use emt_view -s view_name. It’s possible to create custom views on the command line with the fields listed in emt_view -l. I’ll cover this in more detail in a future post.
emt_view is the basic method of accessing the data provided by EMT. There are also other programs such as emt_awk which provide csv output that can be piped to other commands like awk. emt_view is commonly used for analysis and emt_awk is often used by monitoring tools to alert on thresholds. I’ll cover these and other commands in future tutorials.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:11:"Eric Bergen";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:23;a:6:{s:4:"data";s:38:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:5:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:18:"Connector/ODBC 5.1";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:54:"http://dev.mysql.com/downloads/connector/odbc/5.1.html";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:54:"http://dev.mysql.com/downloads/connector/odbc/5.1.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:64:"Connector/ODBC 5.1 (5.1.7 GA, published on Tuesday, 24 Aug 2010)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Tue, 24 Aug 2010 00:00:24 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:277:"Connector/ODBC 5.1 (5.1.7 GA, published on Tuesday, 24 Aug 2010)
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:24;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:41:"What are your favorite MySQL bug reports?";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:33:"http://www.xaprb.com/blog/?p=1988";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:78:"http://www.xaprb.com/blog/2010/08/23/what-are-your-favorite-mysql-bug-reports/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:590:"Bug reports can be fun. They can also be terrible. Either way they can be entertaining. On the Drizzle IRC channel today I saw a couple references to MySQL bug reports: it is stop working and Does not make toast (which reminds me of the Mozilla bug report about the kitchen sink). Got any other favourites1?
1 This one’s for Jay.
Related posts:What are your favorite MySQL replication filtering rules?My new favorite comic: The Adventures of Ace, DBAWhat is your favorite database design book?What are your favorite PostgreSQL performance resources?My favorite wiki is Dokuwiki";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 23:00:51 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:3:"SQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"humor";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1994:"Bug reports can be fun. They can also be terrible. Either way they can be entertaining. On the Drizzle IRC channel today I saw a couple references to MySQL bug reports: it is stop working and Does not make toast (which reminds me of the Mozilla bug report about the kitchen sink). Got any other favourites1?
1 This one’s for Jay.
Related posts:
- What are your favorite MySQL replication filtering rules?
- My new favorite comic: The Adventures of Ace, DBA
- What is your favorite database design book?
- What are your favorite PostgreSQL performance resources?
- My favorite wiki is Dokuwiki
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:22:"Baron Schwartz (xaprb)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:25;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:28:"Yet another DDL dump script";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:34:"http://mysqldb.wordpress.com/?p=39";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:68:"http://mysqldb.wordpress.com/2010/08/23/yet-another-ddl-dump-script/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1469:"Inspired by some recent posts about scripts for parsing mysqldump files, I thought I’d put my own little dump program out there for those looking for alternatives to mysqldump. It’s written in perl and only actually uses the mysqldump utility to dump the data portion of its file dumps. The rest is done via mysql’s internal show commands. Each database is dumped to it’s own directory, where each component in that database is dumped into five subdirs: schemas, routines, views, triggers & data. Inside these subdirs is a file per table and file per proc, etc. The data section is slightly different: as I mentioned it uses mysqldump to dump bulk insert statements into a sql file, one per table. So at any time, any or all components (including data) can be reloaded with a simple redirect or pipe back to mysql client:
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/schemas/*.sql
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/routines/*.sql
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/views/*.sql
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/triggers/*.sql
[user@svr123]$ gunzip -c /path/to/files/dbname/data/*.sql.gz|mysql -D dbname (data files are zipped)
The script is self-explanatory and since I’m pretty much a newb at perl code, I’d love to know if there’s a better way to do any of this stuff. You can find the code here.
Filed under: code, MySQL ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 22:00:57 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:4:"code";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3562:"Inspired by some recent posts about scripts for parsing mysqldump files, I thought I’d put my own little dump program out there for those looking for alternatives to mysqldump. It’s written in perl and only actually uses the mysqldump utility to dump the data portion of its file dumps. The rest is done via mysql’s internal show commands. Each database is dumped to it’s own directory, where each component in that database is dumped into five subdirs: schemas, routines, views, triggers & data. Inside these subdirs is a file per table and file per proc, etc. The data section is slightly different: as I mentioned it uses mysqldump to dump bulk insert statements into a sql file, one per table. So at any time, any or all components (including data) can be reloaded with a simple redirect or pipe back to mysql client:
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/schemas/*.sql
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/routines/*.sql
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/views/*.sql
[user@svr123]$ mysql -D dbname < /path/to/files/dbname/triggers/*.sql
[user@svr123]$ gunzip -c /path/to/files/dbname/data/*.sql.gz|mysql -D dbname
(data files are zipped)
The script is self-explanatory and since I’m pretty much a newb at perl code, I’d love to know if there’s a better way to do any of this stuff. You can find the code here.
Filed under: code, MySQL
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:14:"Ashley Johnson";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:26;a:6:{s:4:"data";s:83:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:20:"MySQL GIS – Part 1";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:33:"http://www.mysqlfanboy.com/?p=314";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:52:"http://www.mysqlfanboy.com/2010/08/mysql-gis-part-1/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:4706:"TweetIn my business (weather) we use lots map based (Geo) information. Almost every table has latitude and longitude. Working with this kind of data can be exciting and frustrating. This should give you a quick start into GIS with MySQL.
“A geographic information system (GIS), or geographical information system, is any system that captures, stores, analyzes, manages, and presents data that are linked to location. In the simplest terms, GIS is the merging of cartography, statistical analysis, and database technology. GIS systems are used in cartography, remote sensing, land surveying, utility management, natural resource management, photogrammetry, geography, urban planning, emergency management, navigation, and localized search engines.” – Wikipedia
GIS / Mapping Systems work with both text data and graphical data. Applications and utilities often blur the lines between the two types and make understanding difficult. Map servers blend raster images, with point or polygon data, and bitmap images to make complete images to display in the user’s client application. For this post I will concentrate on the text type “data”. The type we can index in a MySQL database.
THE SEARCH
After months of reading, [1] I’m writing this post to describes what I have learned about how to get started using GEO coding data as quickly as possible. I found very little piratical information on GIS and MySQL. The MySQL manual covers the functions but doesn’t supply much practical information on GEO. Anders Karlsson wrote a nice and short story about GIS that give me a good start.
The best information has be written by Florin Duroiu in his post titled “Political boundaries overlay in Google maps”. A good part of my post is based on his work.
STEP BY STEP
Below are the detailed needed to to produce a MySQL database with the Points of Interest (POI). This is based on CentOS 5.5 with MySQL 5.1.
yum install gdal
mkdir geo
mkdir data
mkdir data/Oklahoma
cd geo/data/Oklahoma
wget http://downloads.cloudmade.com/north_america/united_states/oklahoma/oklahoma.shapefiles.zip
unzip oklahoma.shapefiles.zip
mysql -e 'create database geo'
ogr2ogr -f "MySQL" MySQL:"geo,user=root,host=localhost,password=" -nln oklahoma_poi -lco engine=MYISAM oklahoma_poi.shp
mysql geo -e 'desc oklahoma_poi'
+----------+--------------+------+-----+---------+----------------+
| Field | Type | Null | Key | Default | Extra |
+----------+--------------+------+-----+---------+----------------+
| OGR_FID | int(11) | NO | PRI | NULL | auto_increment |
| SHAPE | geometry | NO | MUL | NULL | |
| category | varchar(30) | YES | | NULL | |
| name | varchar(113) | YES | | NULL | |
+----------+--------------+------+-----+---------+----------------+
mysql geo -e "select name, category, Y(SHAPE) as lat, X(SHAPE) as lng from oklahoma_poi where NAME like 'School:Putnam%'"
+-------------------------------------+--------------------------------+------------+-------------+
| name | category | lat | lng |
+-------------------------------------+--------------------------------+------------+-------------+
| School:Putnam City West High School | Government and Public Services | 35.492557 | -97.6605975 |
| School:Putnam City North School | Government and Public Services | 35.5892209 | -97.6372648 |
| School:Putnam City School | Government and Public Services | 35.5122794 | -97.6142079 |
| School:Putnam High School | Government and Public Services | 35.5214459 | -97.6086523 |
| School:Putnam Heights Academy | Government and Public Services | 35.5081143 | -97.5397619 |
+-------------------------------------+--------------------------------+------------+-------------+
In a coming set of post I’ll go over:
The “Data” types your will find and how to convert between them.
What data is available and where can you find it?
More examples on what you can do with GIS data.
Viewing our GIS data.
How to collect your own GIS data.
Good and bad examples of searching GIS data.
Optimizing MySQL GIS. Is it really worth using?
[1] Books: GIS for Dummies – Author: Michael N. DeMers – John Wiley & Sons (2009) – ISBN: 0470236825
Open Source GIS: A GRASS GIS Approach. Third Edition.- Author: Markus Neteler and Helena Mitasova – ISBN: 978-0-38735767-6
Web Mapping Illustrated: Using Open Source GIS Toolkits – Author: Tyler Mitchell – ISBN: 9780596008659
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 21:49:55 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:9:{i:0;a:5:{s:4:"data";s:3:"GIS";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:6:"HOW TO";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:17:"Tips & Tricks";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:4:"Data";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:3:"GEO";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:3:"SHP";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:4:"Tips";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:8;a:5:{s:4:"data";s:5:"Tools";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:7465:"Tweet
In my business (weather) we use lots map based (Geo) information. Almost every table has latitude and longitude. Working with this kind of data can be exciting and frustrating. This should give you a quick start into GIS with MySQL.
“A geographic information system (GIS), or geographical information system, is any system that captures, stores, analyzes, manages, and presents data that are linked to location. In the simplest terms, GIS is the merging of cartography, statistical analysis, and database technology. GIS systems are used in cartography, remote sensing, land surveying, utility management, natural resource management, photogrammetry, geography, urban planning, emergency management, navigation, and localized search engines.” – Wikipedia
GIS / Mapping Systems work with both text data and graphical data. Applications and utilities often blur the lines between the two types and make understanding difficult. Map servers blend raster images, with point or polygon data, and bitmap images to make complete images to display in the user’s client application. For this post I will concentrate on the text type “data”. The type we can index in a MySQL database.
THE SEARCH
After months of reading, [1] I’m writing this post to describes what I have learned about how to get started using GEO coding data as quickly as possible. I found very little piratical information on GIS and MySQL. The MySQL manual covers the functions but doesn’t supply much practical information on GEO. Anders Karlsson wrote a nice and short story about GIS that give me a good start.
The best information has be written by Florin Duroiu in his post titled “Political boundaries overlay in Google maps”. A good part of my post is based on his work.
STEP BY STEP
Below are the detailed needed to to produce a MySQL database with the Points of Interest (POI). This is based on CentOS 5.5 with MySQL 5.1.
yum install gdal
mkdir geo
mkdir data
mkdir data/Oklahoma
cd geo/data/Oklahoma
wget http://downloads.cloudmade.com/north_america/united_states/oklahoma/oklahoma.shapefiles.zip
unzip oklahoma.shapefiles.zip
mysql -e 'create database geo'
ogr2ogr -f "MySQL" MySQL:"geo,user=root,host=localhost,password=" -nln oklahoma_poi -lco engine=MYISAM oklahoma_poi.shp
mysql geo -e 'desc oklahoma_poi'
+----------+--------------+------+-----+---------+----------------+
| Field | Type | Null | Key | Default | Extra |
+----------+--------------+------+-----+---------+----------------+
| OGR_FID | int(11) | NO | PRI | NULL | auto_increment |
| SHAPE | geometry | NO | MUL | NULL | |
| category | varchar(30) | YES | | NULL | |
| name | varchar(113) | YES | | NULL | |
+----------+--------------+------+-----+---------+----------------+
mysql geo -e "select name, category, Y(SHAPE) as lat, X(SHAPE) as lng from oklahoma_poi where NAME like 'School:Putnam%'"
+-------------------------------------+--------------------------------+------------+-------------+
| name | category | lat | lng |
+-------------------------------------+--------------------------------+------------+-------------+
| School:Putnam City West High School | Government and Public Services | 35.492557 | -97.6605975 |
| School:Putnam City North School | Government and Public Services | 35.5892209 | -97.6372648 |
| School:Putnam City School | Government and Public Services | 35.5122794 | -97.6142079 |
| School:Putnam High School | Government and Public Services | 35.5214459 | -97.6086523 |
| School:Putnam Heights Academy | Government and Public Services | 35.5081143 | -97.5397619 |
+-------------------------------------+--------------------------------+------------+-------------+
In a coming set of post I’ll go over:
- The “Data” types your will find and how to convert between them.
- What data is available and where can you find it?
- More examples on what you can do with GIS data.
- Viewing our GIS data.
- How to collect your own GIS data.
- Good and bad examples of searching GIS data.
- Optimizing MySQL GIS. Is it really worth using?
[1] Books: GIS for Dummies – Author: Michael N. DeMers – John Wiley & Sons (2009) – ISBN: 0470236825
Open Source GIS: A GRASS GIS Approach. Third Edition.- Author: Markus Neteler and Helena Mitasova – ISBN: 978-0-38735767-6
Web Mapping Illustrated: Using Open Source GIS Toolkits – Author: Tyler Mitchell – ISBN: 9780596008659
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Mark Grennan";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:27;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:46:"InnoDB memory allocation, ulimit, and OpenSUSE";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.mysqlperformanceblog.com/?p=3470";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:92:"http://www.mysqlperformanceblog.com/2010/08/23/innodb-memory-allocation-ulimit-and-opensuse/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:3763:"I recently encountered an interesting case. A customer reported that mysqld crashed on start on OpenSUSE 11.2 kernel 2.6.31.12-0.2-desktop x86_64 with 96 GB RAM when the innodb_buffer_pool_size was set to anything more than 62 GB. I decided to try it with 76 GB. The error message was an assert due to a failed malloc() in ut_malloc_low() in ut/ut0mem.c inside InnoDB source code. InnoDB wraps the majority of its memory allocations in ut_malloc_low(), so to get an idea of the pattern of requested allocations I added a debugging fprintf() to tell me how much was being allocated and whether it was successful.
I discovered something interesting. I expected the allocation to fail on the 76 GB of the buffer pool, due to some weird memory mapping issue and a continuous block of 76 GB not being available. However, that is not what happened. 76 GB buffer was allocated successfully. What was failing is the allocation of 3.37GB after that. What in the world could InnoDB need that was 3.37 GB? There was nothing in the settings that asked for anything close to 3 GB explicitly.
Source code is the ultimate documentation, and I took advantage of that. My good friend GDB guided me to buf_pool_init() in buf/buf0buf.c. There I found the following:
buf_pool->frame_mem = os_mem_alloc_large(
UNIV_PAGE_SIZE * (n_frames + 1),
TRUE, FALSE);
That was the buffer pool itself, the 76 GB of it. And now the buffer pool’s friend:
buf_pool->blocks = ut_malloc(sizeof(buf_block_t) * max_size);
3.6 GB of it!
From the comments in the code (InnoDB code actually has very good comments), max_size is the maximum number of buffer pool pages (16K each), n_frames which is the same thing unless AWE is used, but it was not used, so I did not worry about it.
What shall we call that friend? It is used for storing some meta information about buffer pool pages. The most natural name I could come up with from reading the source code is the blocks array.
Thus we can see that we are allocating another chunk that is in proportion to the setting of innodb_buffer_pool_size for the blocks array. The exact proportions will probably vary from version to version, but roughly about 1 G for every 25 G of the buffer pool. This can become significant in the proper innodb_buffer_pool_size estimations when the system has a lot of RAM and you want to have the largest possible innodb_buffer_pool_size. Do not forget to give the blocks array some room!
While this was an interesting investigation, it nevertheless did not explain why there was not enough room for a 76 GB buffer pool. Even with the extra 3.37 GB allocation, there was still some free memory. Or was there? Maybe some hidden monster was eating it up? I quickly wrote this hack to prove or disprove the monster’s presence.
I verified that I could allocate and initialize two chunks of 40 GB from two separate processes, but not 80 GB from one. In fact, 80GB allocation failed right in malloc(), did not even get to initialization. I tested it with allocating 70 GB concurrently in each process so as to overrun physical memory + swap. Both allocations were successful, one initialized successfully, the other was killed by the OOM kill during initialization.
This smelled like a low ulimit, and sure enough it was. ulimit -m ulimited; ulimit -v unlimited did the magic, and mysqld successfully started with an 80 GB buffer pool. Apparenly OpenSUSE defaults are set in proportion to physical memory to keep the memory-hungry applications from taking the system down. On this particular system (96 GB physical memory, 2 GB swap it decided to set the virtual memory ulimit (-v) to 77.27 GB, and the physical memory (-m) to 80.40 GB).
Entry posted by sasha |
3 comments
Add to: | | | | ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 20:55:52 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:6180:"I recently encountered an interesting case. A customer reported that mysqld crashed on start on OpenSUSE 11.2 kernel 2.6.31.12-0.2-desktop x86_64 with 96 GB RAM when the innodb_buffer_pool_size was set to anything more than 62 GB. I decided to try it with 76 GB. The error message was an assert due to a failed malloc() in ut_malloc_low() in ut/ut0mem.c inside InnoDB source code. InnoDB wraps the majority of its memory allocations in ut_malloc_low(), so to get an idea of the pattern of requested allocations I added a debugging fprintf() to tell me how much was being allocated and whether it was successful.
I discovered something interesting. I expected the allocation to fail on the 76 GB of the buffer pool, due to some weird memory mapping issue and a continuous block of 76 GB not being available. However, that is not what happened. 76 GB buffer was allocated successfully. What was failing is the allocation of 3.37GB after that. What in the world could InnoDB need that was 3.37 GB? There was nothing in the settings that asked for anything close to 3 GB explicitly.
Source code is the ultimate documentation, and I took advantage of that. My good friend GDB guided me to buf_pool_init() in buf/buf0buf.c. There I found the following:
buf_pool->frame_mem = os_mem_alloc_large(
UNIV_PAGE_SIZE * (n_frames + 1),
TRUE, FALSE);
That was the buffer pool itself, the 76 GB of it. And now the buffer pool’s friend:
buf_pool->blocks = ut_malloc(sizeof(buf_block_t) * max_size);
3.6 GB of it!
From the comments in the code (InnoDB code actually has very good comments), max_size is the maximum number of buffer pool pages (16K each), n_frames which is the same thing unless AWE is used, but it was not used, so I did not worry about it.
What shall we call that friend? It is used for storing some meta information about buffer pool pages. The most natural name I could come up with from reading the source code is the blocks array.
Thus we can see that we are allocating another chunk that is in proportion to the setting of innodb_buffer_pool_size for the blocks array. The exact proportions will probably vary from version to version, but roughly about 1 G for every 25 G of the buffer pool. This can become significant in the proper innodb_buffer_pool_size estimations when the system has a lot of RAM and you want to have the largest possible innodb_buffer_pool_size. Do not forget to give the blocks array some room!
While this was an interesting investigation, it nevertheless did not explain why there was not enough room for a 76 GB buffer pool. Even with the extra 3.37 GB allocation, there was still some free memory. Or was there? Maybe some hidden monster was eating it up? I quickly wrote this hack to prove or disprove the monster’s presence.
I verified that I could allocate and initialize two chunks of 40 GB from two separate processes, but not 80 GB from one. In fact, 80GB allocation failed right in malloc(), did not even get to initialization. I tested it with allocating 70 GB concurrently in each process so as to overrun physical memory + swap. Both allocations were successful, one initialized successfully, the other was killed by the OOM kill during initialization.
This smelled like a low ulimit, and sure enough it was. ulimit -m ulimited; ulimit -v unlimited did the magic, and mysqld successfully started with an 80 GB buffer pool. Apparenly OpenSUSE defaults are set in proportion to physical memory to keep the memory-hungry applications from taking the system down. On this particular system (96 GB physical memory, 2 GB swap it decided to set the virtual memory ulimit (-v) to 77.27 GB, and the physical memory (-m) to 80.40 GB).
Entry posted by sasha |
3 comments
Add to: | | | |
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:22:"MySQL Performance Blog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:28;a:6:{s:4:"data";s:53:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:37:"MySQL Cluster 7.1.5 binaries released";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:32:"http://www.clusterdb.com/?p=1246";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:77:"http://www.clusterdb.com/mysql-cluster/mysql-cluster-7-1-5-binaries-released/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:269:"The binary version for MySQL Cluster 7.1.5 has now been made available at http://www.mysql.com/downloads/cluster/
A description of all of the changes (fixes) that have gone into MySQL Cluster 7.1.5 (compared to 7.1.4) can be found in the MySQL Cluster 7.1.5 Change Log.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 17:30:57 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:3:{i:0;a:5:{s:4:"data";s:13:"MySQL Cluster";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:17:"MySQL Cluster 7.1";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:985:"The binary version for MySQL Cluster 7.1.5 has now been made available at http://www.mysql.com/downloads/cluster/
A description of all of the changes (fixes) that have gone into MySQL Cluster 7.1.5 (compared to 7.1.4) can be found in the MySQL Cluster 7.1.5 Change Log.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Andrew Morgan";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:29;a:6:{s:4:"data";s:68:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:47:"Free webinar on MySQL performance this Thursday";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:33:"http://www.xaprb.com/blog/?p=1984";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:85:"http://www.xaprb.com/blog/2010/08/23/free-webinar-on-mysql-performance-this-thursday/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:996:"ODTUG invited me to give a webinar and I said yes, so this Thursday you’re invited to join me as I talk about MySQL performance. We’ve come a very long way towards a MySQL that can perform well on modern hardware, and there really isn’t broad recognition of this. A lot of the best work has gone into the InnoDB “plugin” storage engine, which was announced after my co-authors and I sent High Performance MySQL to the press. I will explain what you should be doing differently now than you did two years ago, and suggest a performance-in-a-nutshell configuration baseline for MySQL that’s quite different from what I’d have said in 2008. You can register for free through GoToWebinar. See you there.
Related posts:Get a free sample chapter of High Performance MySQL Second EditionMySQL: Free Software but not Open SourceSpeaking at NovaRUG on ThursdayHigh Performance MySQL is going to press, againComing soon: High Performance MySQL, Second Edition";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 16:35:26 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:6:{i:0;a:5:{s:4:"data";s:11:"Conferences";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:3:"SQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:6:"InnoDB";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:5:"ODTUG";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:11:"performance";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:7:"Webinar";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2317:"ODTUG invited me to give a webinar and I said yes, so this Thursday you’re invited to join me as I talk about MySQL performance. We’ve come a very long way towards a MySQL that can perform well on modern hardware, and there really isn’t broad recognition of this. A lot of the best work has gone into the InnoDB “plugin” storage engine, which was announced after my co-authors and I sent High Performance MySQL to the press. I will explain what you should be doing differently now than you did two years ago, and suggest a performance-in-a-nutshell configuration baseline for MySQL that’s quite different from what I’d have said in 2008. You can register for free through GoToWebinar. See you there.
Related posts:
- Get a free sample chapter of High Performance MySQL Second Edition
- MySQL: Free Software but not Open Source
- Speaking at NovaRUG on Thursday
- High Performance MySQL is going to press, again
- Coming soon: High Performance MySQL, Second Edition
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:22:"Baron Schwartz (xaprb)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:30;a:6:{s:4:"data";s:73:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:61:"MySQL stored procedure debugging, can I sue for going insane?";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:41:"http://mysqlpreacher.com/wordpress/?p=423";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:103:"http://mysqlpreacher.com/wordpress/2010/08/mysql-stored-procedure-debugging-can-i-sue-for-going-insane/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1971:"Lets paint the picture:
Scenario part 1 : Migrating a couple thousand stored procedures from database technology X to mysql
Scenario part 2 : Legacy system where the people who wrote it left a long time ago
Scenario part 3 : Developers sure can get real creative and invent all kinds of ways to get data (eg: having a stored proc which formulates a big query using concat after going through a bunch of conditions (fair enough), BUT the parts making up the different queries are stored in a table on a database rather than within the stored proc itself) … talk about KIS – Keep it simple!!
Scenario part 4 : This stored proc references 18 tables, 4 views, and another two stored procedures on 5 databases
Now close your eyes and try to imagine that for a few seconds, nah kidding don’t want you to hurt yourself.
I wonder, who’s gonna cover my health insurance if i go crazy? :)
mysql 02:55:47 DEV > call storedprocblahbla(‘I’,'am’,'going’,'crazy’);
ERROR 1052 (23000): Column ‘state_of_mind’ in field list is ambiguous
Sure thats REALLY REALLY helpful thanks :), you know what, lets just mysqldump -d -R -B db1 db2 db3 db4 db5 > /wherever/you/like and grep for column ‘state_of_mind’.
I love a challenge but facing migration of so many stored procs I’d really love to have something “GOOD” to debug them with.
The solutions I found were:
1. Illatis eclipse plugin for $40
works on linux, mac and windows but does show me a lot of:
“An error has occurred when activating this view
com/illatis/parser/lib/k”
2. mydebugger for $50
works on linux and mac via wine
3. dbForge for MySQL for $50
works only on windows due to .net framework (kinda crap because I `hate` using the former)
any other ideas are very welcome, especially opensource ones but I can’t find anything.
I can always try setting up debugging using GDB but would rather try to avoid that.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 15:10:28 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:7:{i:0;a:5:{s:4:"data";s:8:"Advanced";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:9:"debugging";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:4:"mysq";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:5:"procs";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:16:"stored procedure";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:12:"stored procs";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2326:"Lets paint the picture:
Scenario part 1 : Migrating a couple thousand stored procedures from database technology X to mysql
Scenario part 2 : Legacy system where the people who wrote it left a long time ago
Scenario part 3 : Developers sure can get real creative and invent all kinds of ways to get data (eg: having a stored proc which formulates a big query using concat after going through a bunch of conditions (fair enough), BUT the parts making up the different queries are stored in a table on a database rather than within the stored proc itself) … talk about KIS – Keep it simple!!
Scenario part 4 : This stored proc references 18 tables, 4 views, and another two stored procedures on 5 databases
Now close your eyes and try to imagine that for a few seconds, nah kidding don’t want you to hurt yourself.
I wonder, who’s gonna cover my health insurance if i go crazy? :)
mysql 02:55:47 DEV > call storedprocblahbla(‘I’,'am’,'going’,'crazy’);
ERROR 1052 (23000): Column ‘state_of_mind’ in field list is ambiguous
Sure thats REALLY REALLY helpful thanks :), you know what, lets just mysqldump -d -R -B db1 db2 db3 db4 db5 > /wherever/you/like and grep for column ‘state_of_mind’.
I love a challenge but facing migration of so many stored procs I’d really love to have something “GOOD” to debug them with.
The solutions I found were:
1. Illatis eclipse plugin for $40
works on linux, mac and windows but does show me a lot of:
“An error has occurred when activating this view
com/illatis/parser/lib/k”
2. mydebugger for $50
works on linux and mac via wine
3. dbForge for MySQL for $50
works only on windows due to .net framework (kinda crap because I `hate` using the former)
any other ideas are very welcome, especially opensource ones but I can’t find anything.
I can always try setting up debugging using GDB but would rather try to avoid that.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Darren Cassar";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:31;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:52:"Call for presentations for Collaborate 2011 is open!";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:31:"1078 at http://technocation.org";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:73:"http://technocation.org/content/call-presentations-collaborate-2011-open!";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:796:"Collaborate 2011 is a database operational conference to be held in Orlando, Florida April 10th - 14th 2011. Unfortunately, the O'Reilly MySQL Conference has been scheduled for the same week; however, I am working with the conference committee to make sure that speakers that want to attend both are scheduled appropriately (I intend on going to both!). The call for presentations is now open through the end of September, so you have just over a month to submit your proposals. Experience has shown that the best presentations are submitted well in advance of the deadline -- I guess when folks are time-crunched by the deadline they do not make such great abstracts....Speakers whose presentations are accepted receive free admission to the conference and a speaker thank-you gift.
read more";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 15:07:51 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:4:"News";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:11:"Conferences";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1111:"Collaborate 2011 is a database operational conference to be held in Orlando, Florida April 10th - 14th 2011. Unfortunately, the O'Reilly MySQL Conference has been scheduled for the same week; however, I am working with the conference committee to make sure that speakers that want to attend both are scheduled appropriately (I intend on going to both!). The call for presentations is now open through the end of September, so you have just over a month to submit your proposals. Experience has shown that the best presentations are submitted well in advance of the deadline -- I guess when folks are time-crunched by the deadline they do not make such great abstracts....Speakers whose presentations are accepted receive free admission to the conference and a speaker thank-you gift.
read more
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Technocation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:32;a:6:{s:4:"data";s:58:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:44:"Troubleshooting : mysql server has gone away";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:59:"tag:blogger.com,1999:blog-13606853.post-2655140313358542192";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:79:"http://jayant7k.blogspot.com/2010/08/troubleshooting-mysql-server-has-gone.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:409:"When running query against a database the following error can be generated:
ERROR 2006 (HY000) at line NNN: MySQL server has gone away
Where "NNN" is the line number of the script currently being run where the error occurred.
Possible Causes and Resolution
This is a general error which can have a number of possible causes. The one certainty is that the MySQL database is no longer listening on";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Mon, 23 Aug 2010 05:18:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:4:{i:0;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:12:"wait_timeout";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:16:"troubleshooting.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:12:"mysql server";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:622:"When running query against a database the following error can be generated:
ERROR 2006 (HY000) at line NNN: MySQL server has gone away
Where "NNN" is the line number of the script currently being run where the error occurred.
Possible Causes and Resolution
This is a general error which can have a number of possible causes. The one certainty is that the MySQL database is no longer listening on
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Jayant Kumar";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:33;a:6:{s:4:"data";s:73:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:66:"Kontrollbase – new version available with improved analytics";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:30:"http://kontrollsoft.com/?p=763";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:62:"http://feedproxy.google.com/~r/Kontrollsoft/~3/GAYByBNhGfg/763";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:383:"A new version of Kontrollbase – the enterprise monitoring, analytics, reporting, and historical analysis webapp for MySQL database administrators and advanced users of MySQL databases – is available for download. There are several upgrades to the reporting code with improved alert algorithms as well as a new script for auto-archiving of the statistics table based [...]";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sun, 22 Aug 2010 20:39:17 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:7:{i:0;a:5:{s:4:"data";s:12:"announcement";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:9:"analytics";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:12:"kontrollbase";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:10:"monitoring";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:12:"mysql server";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:6:"tuning";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:689:"A new version of Kontrollbase – the enterprise monitoring, analytics, reporting, and historical analysis webapp for MySQL database administrators and advanced users of MySQL databases – is available for download. There are several upgrades to the reporting code with improved alert algorithms as well as a new script for auto-archiving of the statistics table based [...]
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:9:"Matt Reid";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:34;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:43:"PHP @ FrOSCon: the power of mysqlnd plugins";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:32:"http://blog.ulf-wendel.de/?p=293";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:32:"http://blog.ulf-wendel.de/?p=293";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1845:"
Slowly the power of mysqlnd plugins becomes visible. Mysqlnd plugins challenge MySQL Proxy and are often a noteworthy, if not superior, alternative alternative to MySQL Proxy for PHP users. Plugins can do almost anything MySQL Proxy can do - but on the client. Please find details in the slides. The presentation has been given today at the PHP track at FrOSCon.
The power of mysqlnd plugins
View more presentations on mysqlnd.
The biggest news is certainly the work from David Soria Parra. David made it possible to write mysqlnd plugins in PHP! No C coding skills needed. It can’t get any easier. Insipired by the hints given in the C mysqlnd plugin API documentation he and his employer Mayflower decided to develop the “mysqlnd_uh” PECL extension. “mysqlnd_uh” stands for “mysqlnd userhandler”.
class QueryLogger extends MysqlndUhConnection {
public function query($res, $query) {
error_log($query);
return parent::query($res, $query);
}
}
mysqlnd_uh_set_connection_proxy(new QueryLogger());
Assume you have installed a PHP application and you want to know what queries it runs. Using PECL/mysqlnd_uh it is as little as the above code needed to audit the application. Install PECL/mysqlnd_uh, put the code in your auto_prepend_file and find the queries in your error log. This works with every PHP application running on PHP 5.3.3 or newer. It works without touching the application. Not having to change the application itself makes updates much easier. Whenever the application vendor releases a new version you can just install it. Your QueryLogger is not part of the application. It will continue to work after the application update.
Query logging is trivial. Even load balancing or failover should be easy to implement. Future will proof, stay tuned!
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sun, 22 Aug 2010 20:32:31 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:21:"PlanetMySQL (english)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:19:"PlanetPHP (english)";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2784:"
Slowly the power of mysqlnd plugins becomes visible. Mysqlnd plugins challenge MySQL Proxy and are often a noteworthy, if not superior, alternative alternative to MySQL Proxy for PHP users. Plugins can do almost anything MySQL Proxy can do - but on the client. Please find details in the slides. The presentation has been given today at the PHP track at FrOSCon.
The biggest news is certainly the work from David Soria Parra. David made it possible to write mysqlnd plugins in PHP! No C coding skills needed. It can’t get any easier. Insipired by the hints given in the C mysqlnd plugin API documentation he and his employer Mayflower decided to develop the “mysqlnd_uh” PECL extension. “mysqlnd_uh” stands for “mysqlnd userhandler”.
class QueryLogger extends MysqlndUhConnection {
public function query($res, $query) {
error_log($query);
return parent::query($res, $query);
}
}
mysqlnd_uh_set_connection_proxy(new QueryLogger());
Assume you have installed a PHP application and you want to know what queries it runs. Using PECL/mysqlnd_uh it is as little as the above code needed to audit the application. Install PECL/mysqlnd_uh, put the code in your auto_prepend_file
and find the queries in your error log. This works with every PHP application running on PHP 5.3.3 or newer. It works without touching the application. Not having to change the application itself makes updates much easier. Whenever the application vendor releases a new version you can just install it. Your QueryLogger is not part of the application. It will continue to work after the application update.
Query logging is trivial. Even load balancing or failover should be easy to implement. Future will proof, stay tuned!
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:10:"Ulf Wendel";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:35;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:21:"Moving on from Oracle";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:58:"http://blogs.sun.com/LinuxJedi/entry/moving_on_from_oracle";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:58:"http://blogs.sun.com/LinuxJedi/entry/moving_on_from_oracle";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:763:"This week will be my final week at Oracle. I am very much going to miss the great team I work with but due to many other reasons I have decided to move on.
Where am I going?
I'm sure several people already know, if you don't, this will be revealed soon. It will still be related to MySQL, but not so much MySQL Cluster. I do fully intend to still write patches for MySQL and MySQL Cluster in my spare time and I'll still probably be lurking in the MySQL Cluster mailing list and forums.
What does this mean for this blog?
In all likelihood it will become stale very quickly (worst case it will be removed). I will no longer be able to edit it.
Will there be a new blog?
Yes, at some point soon. Watch out on Planet MySQL.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sun, 22 Aug 2010 19:41:56 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1085:"This week will be my final week at Oracle. I am very much going to miss the great team I work with but due to many other reasons I have decided to move on.
Where am I going?
I'm sure several people already know, if you don't, this will be revealed soon. It will still be related to MySQL, but not so much MySQL Cluster. I do fully intend to still write patches for MySQL and MySQL Cluster in my spare time and I'll still probably be lurking in the MySQL Cluster mailing list and forums.
What does this mean for this blog?
In all likelihood it will become stale very quickly (worst case it will be removed). I will no longer be able to edit it.
Will there be a new blog?
Yes, at some point soon. Watch out on Planet MySQL.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:16:"Andrew Hutchings";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:36;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:22:"LinuxJedi, the new era";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:32:"http://blog.linuxjedi.co.uk/?p=1";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:31:"http://www.linuxjedi.co.uk/?p=1";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:358:"I have recently moved on from my roll as a Senior MySQL Support Engineer at Oracle to a Drizzle Developer at Rackspace. As such my previous blog is now sitting idle so this is the replacement.
This new blog may have much less MySQL Cluster content, but I do hope to continue the same quality (I shall leave you to judge on how high that is!) of blog posts.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sun, 22 Aug 2010 18:38:14 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:7:"Drizzle";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:668:"I have recently moved on from my roll as a Senior MySQL Support Engineer at Oracle to a Drizzle Developer at Rackspace. As such my previous blog is now sitting idle so this is the replacement.
This new blog may have much less MySQL Cluster content, but I do hope to continue the same quality (I shall leave you to judge on how high that is!) of blog posts.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:16:"Andrew Hutchings";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:37;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:30:"Yesterday I heckled a speaker.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:41:"http://mtocker.livejournal.com/52399.html";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:41:"http://mtocker.livejournal.com/52399.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:11828:"It's frustrating seeing examples of MySQL being slow as an example of why you should use NoSQL. If you have an invested interest[1] in comparing two technologies that are already apples to oranges, the least you can do is optimize both. If you can't do it, don't share it.
This came out of a talk on Cassandra. "MySQL" is not on the slide, but it was mentioned in reference to MySQL:
SELECT * FROM tweets WHERE user_id IN (SELECT follower FROM followers WHERE user_id = ?) order by time_tweeted DESC LIMIT 40;
Let me simulate that for you:
CREATE TABLE `tweets` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`user_id` int(11) NOT NULL,
`info` text,
`time_tweeted` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
INDEX (time_tweeted),
INDEX (user_id)
) ENGINE=InnoDB;
CREATE TABLE `followers` (
`user_id` int(11) NOT NULL,
`follower` int(11) NOT NULL,
PRIMARY KEY (`user_id`,`follower`)
) ENGINE=InnoDB;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM dual;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT ignore into followers (user_id, follower) SELECT floor(rand()*10000), floor(rand()*10000) FROM tweets;
mysql> select count(*) from tweets;
+----------+
| count(*) |
+----------+
| 65536 |
+----------+
1 row in set (0.03 sec)
mysql> select count(*) from followers; # there are some duplicates.
+----------+
| count(*) |
+----------+
| 65521 |
+----------+
1 row in set (0.03 sec)
select count(*) from followers where user_id = 55; <-- Ok, found a user
+----------+
| count(*) |
+----------+
| 4 |
+----------+
1 row in set (0.00 sec)
mysql> EXPLAIN SELECT * FROM tweets WHERE user_id IN (SELECT follower FROM followers WHERE user_id = 55) order by time_tweeted DESC LIMIT 40\G
*************************** 1. row ***************************
id: 1
select_type: PRIMARY
table: tweets
type: index
possible_keys: NULL
key: time_tweeted
key_len: 4
ref: NULL
rows: 40
Extra: Using where
*************************** 2. row ***************************
id: 2
select_type: DEPENDENT SUBQUERY
table: followers
type: eq_ref
possible_keys: PRIMARY
key: PRIMARY
key_len: 8
ref: const,func
rows: 1
Extra: Using index
2 rows in set (0.00 sec)
I should note that even though it says "40 rows" in the index scan, it's really a lot more, due to a bug I've previously reported:
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 160 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 142 |
| Handler_read_first | 56 |
| Handler_read_key | 131364 |
| Handler_read_next | 900495 |
| Handler_read_prev | 0 |
| Handler_read_rnd | 42 |
| Handler_read_rnd_next | 750736 |
| Handler_rollback | 7 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 1454583 |
+----------------------------+---------+
15 rows in set (0.00 sec)
.. Run query ..
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 161 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 142 |
| Handler_read_first | 56 |
| Handler_read_key | 218955 | <--- Something like 87591 rows!
| Handler_read_next | 900495 |
| Handler_read_prev | 43793 |
| Handler_read_rnd | 42 |
| Handler_read_rnd_next | 750736 |
| Handler_rollback | 7 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 1454583 |
+----------------------------+---------+
15 rows in set (0.00 sec)
And repeating to get a profile shows the awesome dependent subquery:
mysql> show profile for query 1;
+----------------------+----------+
| Status | Duration |
+----------------------+----------+
| starting | 0.000099 |
| checking permissions | 0.000008 |
| checking permissions | 0.000010 |
| Opening tables | 0.000034 |
| System lock | 0.000015 |
| init | 0.000045 |
| optimizing | 0.000012 |
| statistics | 0.000013 |
| preparing | 0.000013 |
| executing | 0.000004 |
| Sorting result | 0.000007 |
| Sending data | 0.000135 |
| optimizing | 0.000016 |
| statistics | 0.000057 |
| preparing | 0.000020 |
| executing | 0.000005 |
| Sending data | 0.000036 |
| executing | 0.000005 |
..
| Sending data | 0.000027 |
| executing | 0.000005 |
..
| end | 0.000007 |
| query end | 0.000005 |
| freeing items | 0.000149 |
| logging slow query | 0.000006 |
| cleaning up | 0.000006 |
+----------------------+----------+
161966 rows in set (1.33 sec)
The problems with this query are:
The IN() subquery can not be optimized. This is one of the optimizations that MySQL fails at. The secret is that even though MySQL has subqueries, there are few a DBA will recommend using in production. Please pretend they don't exist in comparisons.
Assuming a sort is used (it isn't) ID should be monotonic and ordering by it _is_ the better choice (assuming it is a PK/InnoDB). If you could get a plan to access followers first (as described) and then sort by posted_date on tweets, the data is going to be pre-sorted in the wrong order - since all newer tweets have newer dates. This is the worst case situation to sort.
The visual description of how the query executes in MySQL is incorrect. The tables clearly join in the complete opposite order, and there's no sort. The dependent subquery is why MySQL sucks here.
The query that should have been a join. i.e.:
mysql> EXPLAIN SELECT tweets.* FROM followers INNER JOIN tweets ON tweets.user_id=followers.user_id WHERE followers.user_id = 55 ORDER BY id DESC\G
*************************** 1. row ***************************
id: 1
select_type: SIMPLE
table: tweets
type: ref
possible_keys: user_id
key: user_id
key_len: 4
ref: const
rows: 111
Extra: Using where
*************************** 2. row ***************************
id: 1
select_type: SIMPLE
table: followers
type: ref
possible_keys: PRIMARY
key: PRIMARY
key_len: 4
ref: const
rows: 4
Extra: Using index
2 rows in set (0.00 sec)
Note: This might be a bit difficult to read if you're not familiar with subtle optimizations. The followers table appears second but is id=1, so the filter on followers is applied.
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 75 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 60 |
| Handler_read_first | 31 |
| Handler_read_key | 324231 |
| Handler_read_next | 2198205 |
| Handler_read_prev | 162173 |
| Handler_read_rnd | 444 |
| Handler_read_rnd_next | 2529171 |
| Handler_rollback | 0 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 2579264 |
+----------------------------+---------+
15 rows in set (0.00 sec)
.. run query ..
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 76 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 60 |
| Handler_read_first | 31 |
| Handler_read_key | 324347 | <-- +116
| Handler_read_next | 2198649 | <-- +444
| Handler_read_prev | 162284 | <-- +111
| Handler_read_rnd | 444 |
| Handler_read_rnd_next | 2529171 |
| Handler_rollback | 0 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 2579264 |
+----------------------------+---------+
15 rows in set (0.01 sec)
And show profiles result:
mysql> show profile for query 13;
+----------------------+----------+
| Status | Duration |
+----------------------+----------+
| starting | 0.000136 |
| checking permissions | 0.000007 |
| checking permissions | 0.000007 |
| Opening tables | 0.000069 |
| System lock | 0.000014 |
| init | 0.000028 |
| optimizing | 0.000016 |
| statistics | 0.000095 |
| preparing | 0.000024 |
| executing | 0.000004 |
| Sorting result | 0.000005 |
| Sending data | 0.003517 |
| end | 0.000006 |
| query end | 0.000003 |
| freeing items | 0.000035 |
| logging slow query | 0.000004 |
| cleaning up | 0.000004 |
+----------------------+----------+
17 rows in set (0.01 sec)
I want to learn more about Cassandra. I want speakers to be able to apply context by comparing to something people are familiar with, but sheesh... you've got to get your descriptions right.
I have heard people tell me MySQL is difficult to optimize. It is true, but it's not an excuse. If you really are at "massive scale" you can afford to get a second opinion.
[1] I wouldn't heckle someone who was neutral and made the mistake accidentally.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sun, 22 Aug 2010 10:01:57 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:4:"rant";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:12357:"It's frustrating seeing examples of MySQL being slow as an example of why you should use NoSQL. If you have an invested interest[1] in comparing two technologies that are already apples to oranges, the least you can do is optimize both. If you can't do it, don't share it.
This came out of a talk on Cassandra. "MySQL" is not on the slide, but it was mentioned in reference to MySQL:
SELECT * FROM tweets WHERE user_id IN (SELECT follower FROM followers WHERE user_id = ?) order by time_tweeted DESC LIMIT 40;
Let me simulate that for you:
CREATE TABLE `tweets` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`user_id` int(11) NOT NULL,
`info` text,
`time_tweeted` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
INDEX (time_tweeted),
INDEX (user_id)
) ENGINE=InnoDB;
CREATE TABLE `followers` (
`user_id` int(11) NOT NULL,
`follower` int(11) NOT NULL,
PRIMARY KEY (`user_id`,`follower`)
) ENGINE=InnoDB;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM dual;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT INTO tweets (user_id, info) SELECT floor(rand()*10000), REPEAT('a', 160) FROM tweets;
INSERT ignore into followers (user_id, follower) SELECT floor(rand()*10000), floor(rand()*10000) FROM tweets;
mysql> select count(*) from tweets;
+----------+
| count(*) |
+----------+
| 65536 |
+----------+
1 row in set (0.03 sec)
mysql> select count(*) from followers; # there are some duplicates.
+----------+
| count(*) |
+----------+
| 65521 |
+----------+
1 row in set (0.03 sec)
select count(*) from followers where user_id = 55; <-- Ok, found a user
+----------+
| count(*) |
+----------+
| 4 |
+----------+
1 row in set (0.00 sec)
mysql> EXPLAIN SELECT * FROM tweets WHERE user_id IN (SELECT follower FROM followers WHERE user_id = 55) order by time_tweeted DESC LIMIT 40\G
*************************** 1. row ***************************
id: 1
select_type: PRIMARY
table: tweets
type: index
possible_keys: NULL
key: time_tweeted
key_len: 4
ref: NULL
rows: 40
Extra: Using where
*************************** 2. row ***************************
id: 2
select_type: DEPENDENT SUBQUERY
table: followers
type: eq_ref
possible_keys: PRIMARY
key: PRIMARY
key_len: 8
ref: const,func
rows: 1
Extra: Using index
2 rows in set (0.00 sec)
I should note that even though it says "40 rows" in the index scan, it's really a lot more, due to a bug I've previously reported:
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 160 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 142 |
| Handler_read_first | 56 |
| Handler_read_key | 131364 |
| Handler_read_next | 900495 |
| Handler_read_prev | 0 |
| Handler_read_rnd | 42 |
| Handler_read_rnd_next | 750736 |
| Handler_rollback | 7 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 1454583 |
+----------------------------+---------+
15 rows in set (0.00 sec)
.. Run query ..
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 161 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 142 |
| Handler_read_first | 56 |
| Handler_read_key | 218955 | <--- Something like 87591 rows!
| Handler_read_next | 900495 |
| Handler_read_prev | 43793 |
| Handler_read_rnd | 42 |
| Handler_read_rnd_next | 750736 |
| Handler_rollback | 7 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 1454583 |
+----------------------------+---------+
15 rows in set (0.00 sec)
And repeating to get a profile shows the awesome dependent subquery:
mysql> show profile for query 1;
+----------------------+----------+
| Status | Duration |
+----------------------+----------+
| starting | 0.000099 |
| checking permissions | 0.000008 |
| checking permissions | 0.000010 |
| Opening tables | 0.000034 |
| System lock | 0.000015 |
| init | 0.000045 |
| optimizing | 0.000012 |
| statistics | 0.000013 |
| preparing | 0.000013 |
| executing | 0.000004 |
| Sorting result | 0.000007 |
| Sending data | 0.000135 |
| optimizing | 0.000016 |
| statistics | 0.000057 |
| preparing | 0.000020 |
| executing | 0.000005 |
| Sending data | 0.000036 |
| executing | 0.000005 |
..
| Sending data | 0.000027 |
| executing | 0.000005 |
..
| end | 0.000007 |
| query end | 0.000005 |
| freeing items | 0.000149 |
| logging slow query | 0.000006 |
| cleaning up | 0.000006 |
+----------------------+----------+
161966 rows in set (1.33 sec)
The problems with this query are:
- The IN() subquery can not be optimized. This is one of the optimizations that MySQL fails at. The secret is that even though MySQL has subqueries, there are few a DBA will recommend using in production. Please pretend they don't exist in comparisons.
- Assuming a sort is used (it isn't) ID should be monotonic and ordering by it _is_ the better choice (assuming it is a PK/InnoDB). If you could get a plan to access followers first (as described) and then sort by posted_date on tweets, the data is going to be pre-sorted in the wrong order - since all newer tweets have newer dates. This is the worst case situation to sort.
- The visual description of how the query executes in MySQL is incorrect. The tables clearly join in the complete opposite order, and there's no sort. The dependent subquery is why MySQL sucks here.
The query that should have been a join. i.e.:
mysql> EXPLAIN SELECT tweets.* FROM followers INNER JOIN tweets ON tweets.user_id=followers.user_id WHERE followers.user_id = 55 ORDER BY id DESC\G
*************************** 1. row ***************************
id: 1
select_type: SIMPLE
table: tweets
type: ref
possible_keys: user_id
key: user_id
key_len: 4
ref: const
rows: 111
Extra: Using where
*************************** 2. row ***************************
id: 1
select_type: SIMPLE
table: followers
type: ref
possible_keys: PRIMARY
key: PRIMARY
key_len: 4
ref: const
rows: 4
Extra: Using index
2 rows in set (0.00 sec)
Note: This might be a bit difficult to read if you're not familiar with subtle optimizations. The followers table appears second but is id=1, so the filter on followers is applied.
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 75 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 60 |
| Handler_read_first | 31 |
| Handler_read_key | 324231 |
| Handler_read_next | 2198205 |
| Handler_read_prev | 162173 |
| Handler_read_rnd | 444 |
| Handler_read_rnd_next | 2529171 |
| Handler_rollback | 0 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 2579264 |
+----------------------------+---------+
15 rows in set (0.00 sec)
.. run query ..
mysql> show session status like 'Handler%';
+----------------------------+---------+
| Variable_name | Value |
+----------------------------+---------+
| Handler_commit | 76 |
| Handler_delete | 0 |
| Handler_discover | 0 |
| Handler_prepare | 60 |
| Handler_read_first | 31 |
| Handler_read_key | 324347 | <-- +116
| Handler_read_next | 2198649 | <-- +444
| Handler_read_prev | 162284 | <-- +111
| Handler_read_rnd | 444 |
| Handler_read_rnd_next | 2529171 |
| Handler_rollback | 0 |
| Handler_savepoint | 0 |
| Handler_savepoint_rollback | 0 |
| Handler_update | 0 |
| Handler_write | 2579264 |
+----------------------------+---------+
15 rows in set (0.01 sec)
And show profiles result:
mysql> show profile for query 13;
+----------------------+----------+
| Status | Duration |
+----------------------+----------+
| starting | 0.000136 |
| checking permissions | 0.000007 |
| checking permissions | 0.000007 |
| Opening tables | 0.000069 |
| System lock | 0.000014 |
| init | 0.000028 |
| optimizing | 0.000016 |
| statistics | 0.000095 |
| preparing | 0.000024 |
| executing | 0.000004 |
| Sorting result | 0.000005 |
| Sending data | 0.003517 |
| end | 0.000006 |
| query end | 0.000003 |
| freeing items | 0.000035 |
| logging slow query | 0.000004 |
| cleaning up | 0.000004 |
+----------------------+----------+
17 rows in set (0.01 sec)
I want to learn more about Cassandra. I want speakers to be able to apply context by comparing to something people are familiar with, but sheesh... you've got to get your descriptions right.
I have heard people tell me MySQL is difficult to optimize. It is true, but it's not an excuse. If you really are at "massive scale" you can afford to get a second opinion.
[1] I wouldn't heckle someone who was neutral and made the mistake accidentally.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Morgan Tocker";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:38;a:6:{s:4:"data";s:68:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:65:"Using LVM snapshot filesystems for development database instances";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:26:"http://blog.wl0.org/?p=321";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:94:"http://blog.wl0.org/2010/08/using-lvm-snapshot-filesystems-for-development-database-instances/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:6967:"The Problem
Developers often need to have a development database copy of the live production system you are using in able to allow them to test their code and to test new functionality and make schema changes to the database for this new functionality to work.
That’s normal and happens everywhere. A typical DBA task is to make a copy of the live system, sometimes to remove any confidential or sensitive information which perhaps the development database users should not be able to see, and then give them access to this development instance. The developers then “hack away”, changing their code and perhaps things in the database until they are ready to put these new changes into production when they then come along and discuss how to apply these changes into the live systems.
Once the development database has been created it soon becomes stale so often the developers want a new up to date copy to be made to “simplify” their testing.
This is all fine until the database size begins to grow and this process of dumping and copying the data for the developers takes hours rather than minutes and therefore can only be done on a daily or weekly basis.
I have recently been experimenting with the use of mounting the development database instance on an LVM snapshot of the original filesystem where the live system is running. This procedure does not have to be Linux specific but should work with any OS or storage which provides a facility to make a filesystem snapshot based on the contents of another filesystem.
So what does this mean in practice?
Previous Behaviour:
Situation: server1 has a live production copy of the database. We want to make a copy to server2. server2 is already prepared with a configuration which will work based on the copy of the production data.
Procedure: stop server1, copy the filesystem holding the database to server2, start server1. start server2. [the copy procedure takes hours.]
New Behaviour:
Situation: server1 has a live production copy of the database (probably via a slave), and space/memory for a second development instance to run concurrently on the same server.
Procedure: stop server1 (live instance), make a LVM snapshot of the live filesystem (the snapshot size can be much smaller than the live filesystem size), start server1 (live instance), start server1 (development instance). [the copy procedure takes just a few seconds.]
Since I do this with the live system being a slave, I tend to also include a routine to disable replication information on the snapshot filesystem by removing the appropriate files. It may also be necessary adjust the grants on the dev-instance so that it is appropriate for the new set of db users.
To all intents and purposes when you login to the development instance it looks like an up to date copy of live system. You can make as many changes as you like as long as the number of disk blocks on the snapshot which get changed don’t exceed the snapshot size. At this point the snapshot filesystem becomes invalid and mysqld is unable to access it. Mysqld gets a bit upset about this, but you just kill it and then go and rebuild the instance again if this happens – it only takes a few seconds.
This works pretty well and speeds things up for the developers. I can create a new development environment from the live system in seconds rather than hours. The disk storage requirements also tend to drop significantly. It also helps the devs. If you do a daily refresh of this development instance then it allows the developers to test any schema changes which will be needed to be applied to the live system much more easily as “going back to the current live state” is so easy.
So if you haven’t done something like this it might be worth giving it a go.
This is an example of the output from a script I’m currently using:
[root@myhost ~]# clone_instance -s40G -d live-instance dev-instance
Aug 22 10:57:00 myhost clone_instance[16405] Cloning live-instance to dev-instance with a snapshot volume of size 40G
Aug 22 10:57:00 myhost clone_instance[16405] /mysql/live-instance is mounted as expected
Aug 22 10:57:00 myhost clone_instance[16405] Device /dev/volgroup1/live-instance is mounted on /mysql/live-instance, having volume group: volgroup1, logical volume: live-instance
Aug 22 10:57:00 myhost clone_instance[16405] live-instance is defined in /etc/my.cnf [mysqld1]
Aug 22 10:57:00 myhost clone_instance[16405] dev-instance is defined in /etc/my.cnf [mysqld2]
Aug 22 10:57:00 myhost clone_instance[16405] Found defaults file /root/.my-live.cnf needed to shutdown live-instance
Aug 22 10:57:00 myhost clone_instance[16405] Going to viciously kill any processes using files under mount point: /mysql/dev-instance
Aug 22 10:57:02 myhost clone_instance[16405] Unmounting /mysql/dev-instance
Aug 22 10:57:02 myhost clone_instance[16405] Removing existing SNAPSHOT LV /dev/volgroup1/dev-instance
Logical volume "dev-instance" successfully removed
Aug 22 10:57:03 myhost clone_instance[16405] SNAPSHOT LV /dev/volgroup1/dev-instance removed
Aug 22 10:57:03 myhost clone_instance[16405] Shutting down live-instance [mysqld1] using mysqldmin and defaults file /root/.my-live.cnf (as 'mysqld_multi stop 1' does not seem work properly)
Aug 22 11:02:34 myhost clone_instance[16405] Creating new snapshot LV dev-instance (40G) based on /dev/volgroup1/live-instance
Logical volume "dev-instance" created
Aug 22 11:02:35 myhost clone_instance[16405] Restarting live-instance [mysqld1] using mysqld_multi start 1
Aug 22 11:02:35 myhost clone_instance[16405] Mounting SNAPSHOT LV /dev/volgroup1/dev-instance on /mysql/dev-instance
Aug 22 11:02:35 myhost clone_instance[16405] Cleaning up log files on SNAPSHOT LV
Aug 22 11:02:35 myhost clone_instance[16405] Removing replication information from SNAPSHOT LV
Aug 22 11:02:35 myhost clone_instance[16405] Found defaults file /root/.my-dev.cnf needed to access dev-instance
Aug 22 11:02:35 myhost clone_instance[16405] Starting SNAPSHOT DB instance dev-instance [mysqld2] using: 'mysqld_multi start 2'
Aug 22 11:02:35 myhost clone_instance[16405] Clone procedure complete.
[root@myhost ~]# df -h
Filesystem Size Used Avail Use% Mounted on
...
/dev/mapper/volgroup1-live--instance
160G 135G 26G 84% /mysql/live-instance
/dev/mapper/volgroup1-dev--instance
160G 134G 27G 84% /mysql/dev-instance
[root@myhost ~]# lvs
LV VG Attr LSize Origin Snap% Move Log Copy% Convert
live-instance volgroup1 owi-ao 160.00G
dev-instance volgroup1 swi-ao 40.00G live-instance 2.06
[root@myhost ~]#
The longest part of the procedure is shutting down the active live slave. The rest of the time is insignificant.
The “dev experience” when using this dev-instance is just as before. The instance works, can be modified and behaves just as you would expect.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sun, 22 Aug 2010 09:28:40 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:6:{i:0;a:5:{s:4:"data";s:9:"Databases";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:14:"clone database";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:3:"dev";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:5:"linux";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:3:"LVM";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:8:"snapshot";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:7747:"The Problem
Developers often need to have a development database copy of the live production system you are using in able to allow them to test their code and to test new functionality and make schema changes to the database for this new functionality to work.
That’s normal and happens everywhere. A typical DBA task is to make a copy of the live system, sometimes to remove any confidential or sensitive information which perhaps the development database users should not be able to see, and then give them access to this development instance. The developers then “hack away”, changing their code and perhaps things in the database until they are ready to put these new changes into production when they then come along and discuss how to apply these changes into the live systems.
Once the development database has been created it soon becomes stale so often the developers want a new up to date copy to be made to “simplify” their testing.
This is all fine until the database size begins to grow and this process of dumping and copying the data for the developers takes hours rather than minutes and therefore can only be done on a daily or weekly basis.
I have recently been experimenting with the use of mounting the development database instance on an LVM snapshot of the original filesystem where the live system is running. This procedure does not have to be Linux specific but should work with any OS or storage which provides a facility to make a filesystem snapshot based on the contents of another filesystem.
So what does this mean in practice?
Previous Behaviour:
Situation: server1 has a live production copy of the database. We want to make a copy to server2. server2 is already prepared with a configuration which will work based on the copy of the production data.
Procedure: stop server1, copy the filesystem holding the database to server2, start server1. start server2. [the copy procedure takes hours.]
New Behaviour:
Situation: server1 has a live production copy of the database (probably via a slave), and space/memory for a second development instance to run concurrently on the same server.
Procedure: stop server1 (live instance), make a LVM snapshot of the live filesystem (the snapshot size can be much smaller than the live filesystem size), start server1 (live instance), start server1 (development instance). [the copy procedure takes just a few seconds.]
Since I do this with the live system being a slave, I tend to also include a routine to disable replication information on the snapshot filesystem by removing the appropriate files. It may also be necessary adjust the grants on the dev-instance so that it is appropriate for the new set of db users.
To all intents and purposes when you login to the development instance it looks like an up to date copy of live system. You can make as many changes as you like as long as the number of disk blocks on the snapshot which get changed don’t exceed the snapshot size. At this point the snapshot filesystem becomes invalid and mysqld is unable to access it. Mysqld gets a bit upset about this, but you just kill it and then go and rebuild the instance again if this happens – it only takes a few seconds.
This works pretty well and speeds things up for the developers. I can create a new development environment from the live system in seconds rather than hours. The disk storage requirements also tend to drop significantly. It also helps the devs. If you do a daily refresh of this development instance then it allows the developers to test any schema changes which will be needed to be applied to the live system much more easily as “going back to the current live state” is so easy.
So if you haven’t done something like this it might be worth giving it a go.
This is an example of the output from a script I’m currently using:
[root@myhost ~]# clone_instance -s40G -d live-instance dev-instance
Aug 22 10:57:00 myhost clone_instance[16405] Cloning live-instance to dev-instance with a snapshot volume of size 40G
Aug 22 10:57:00 myhost clone_instance[16405] /mysql/live-instance is mounted as expected
Aug 22 10:57:00 myhost clone_instance[16405] Device /dev/volgroup1/live-instance is mounted on /mysql/live-instance, having volume group: volgroup1, logical volume: live-instance
Aug 22 10:57:00 myhost clone_instance[16405] live-instance is defined in /etc/my.cnf [mysqld1]
Aug 22 10:57:00 myhost clone_instance[16405] dev-instance is defined in /etc/my.cnf [mysqld2]
Aug 22 10:57:00 myhost clone_instance[16405] Found defaults file /root/.my-live.cnf needed to shutdown live-instance
Aug 22 10:57:00 myhost clone_instance[16405] Going to viciously kill any processes using files under mount point: /mysql/dev-instance
Aug 22 10:57:02 myhost clone_instance[16405] Unmounting /mysql/dev-instance
Aug 22 10:57:02 myhost clone_instance[16405] Removing existing SNAPSHOT LV /dev/volgroup1/dev-instance
Logical volume "dev-instance" successfully removed
Aug 22 10:57:03 myhost clone_instance[16405] SNAPSHOT LV /dev/volgroup1/dev-instance removed
Aug 22 10:57:03 myhost clone_instance[16405] Shutting down live-instance [mysqld1] using mysqldmin and defaults file /root/.my-live.cnf (as 'mysqld_multi stop 1' does not seem work properly)
Aug 22 11:02:34 myhost clone_instance[16405] Creating new snapshot LV dev-instance (40G) based on /dev/volgroup1/live-instance
Logical volume "dev-instance" created
Aug 22 11:02:35 myhost clone_instance[16405] Restarting live-instance [mysqld1] using mysqld_multi start 1
Aug 22 11:02:35 myhost clone_instance[16405] Mounting SNAPSHOT LV /dev/volgroup1/dev-instance on /mysql/dev-instance
Aug 22 11:02:35 myhost clone_instance[16405] Cleaning up log files on SNAPSHOT LV
Aug 22 11:02:35 myhost clone_instance[16405] Removing replication information from SNAPSHOT LV
Aug 22 11:02:35 myhost clone_instance[16405] Found defaults file /root/.my-dev.cnf needed to access dev-instance
Aug 22 11:02:35 myhost clone_instance[16405] Starting SNAPSHOT DB instance dev-instance [mysqld2] using: 'mysqld_multi start 2'
Aug 22 11:02:35 myhost clone_instance[16405] Clone procedure complete.
[root@myhost ~]# df -h
Filesystem Size Used Avail Use% Mounted on
...
/dev/mapper/volgroup1-live--instance
160G 135G 26G 84% /mysql/live-instance
/dev/mapper/volgroup1-dev--instance
160G 134G 27G 84% /mysql/dev-instance
[root@myhost ~]# lvs
LV VG Attr LSize Origin Snap% Move Log Copy% Convert
live-instance volgroup1 owi-ao 160.00G
dev-instance volgroup1 swi-ao 40.00G live-instance 2.06
[root@myhost ~]#
The longest part of the procedure is shutting down the active live slave. The rest of the time is insignificant.
The “dev experience” when using this dev-instance is just as before. The instance works, can be modified and behaves just as you would expect.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:10:"Simon Mudd";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:39;a:6:{s:4:"data";s:53:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:95:"MONyog 4.0 introduces MySQL configuration tracking, improved customization and better stability";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:34:"http://www.webyog.com/blog/?p=1998";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:133:"http://www.webyog.com/blog/2010/08/21/monyog-4-0-introduces-mysql-configuration-tracking-improved-customization-and-better-stability/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:3285:"Hi,
We are delighted to unveil MONyog 4.0 — a major new release. Listed below are overviews of some of the major features.
Tracking changes to your MySQL configuration
Maintaining server configuration and tracking changes to it plays a vital role in the maintenance of MySQL servers. DBAs may be responsible for hundreds of servers and keeping an eye on the configuration settings for all of them could be difficult to say the least. With MONyog 4.0 now you can compare global MySQL configuration of multiple servers side-by-side, with all changes highlighted so that differences are visually discernible at a glance. Wondering why server A isn’t performing as well as server B when they share the exact same load? The answer could lie in the configuration files!
What’s more is that MONyog now lets you track changes to the global configuration of your MySQL server no matter if the configuration parameters were specified in my.ini/my.cnf, are server defaults or if somebody with SUPER privilege has executed a SET GLOBAL statement.
Improved customization framework
Many MONyog users have long since complained that the customization feature was complex and hard to use: One had to be well-versed in object-oriented JavaScript concepts to even make a minor change. Moreover, unless you manually backed up your customizations, you’d loose all of them if you decided to upgrade! That was motivation enough for us to redesign the entire framework.
With MONyog 4.0 knowledge of writing basic JavaScript functions is more than sufficient to customize any Monitor/Advisor or roll out your very own set. A simple form-based interface with verbose instructions makes it very easy to define a new Monitor/Advisor.
We understand that not every user will make use of the 220+ Monitors/Advisors that we ship with MONyog. So why waste all the computing power? You can now disable a subset of the Monitors/Advisors and keep only the ones you use. If you change your mind later, you can always re-enable them again.
MONyog 4.0 provides a unique customization interface. Now you won’t have to worry about loosing your changes, making mistakes (you can always revert to the original!), or feel scared to experiment.
Improved stability and lots of minor features and bug fixes
As always, this release too contains a lot of bug fixes. MONyog has undergone severe stress testing and is monitoring 200 without issues in our test environment now. From this release we have reduced the number of Linux binaries. In version 3.72 we introduced packages specific to glibc versions. It was necessary at that time to ensure stability on both older Linux distros (RHEL3-generation) as well as the most recent distros using glibc version 2.5. But we never liked it ourselves. It could confuse users – and sometimes did. Since then we have been researching and testing different packaging concepts so that a single binary will do the job on all distros. This release contains a single 32 and a single 64 binary – each with the option of a .tar.gz archive and a RPM installer.
MONyog customers can download the latest installers from Webyog’s Customer Portal.
We are very excited about this release, and hope that you will like it. We would love to hear from you.
Regards,
Team MONyog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sat, 21 Aug 2010 13:21:19 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:3:{i:0;a:5:{s:4:"data";s:6:"MONyog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:8:"Releases";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3669:"Hi,
We are delighted to unveil MONyog 4.0 — a major new release. Listed below are overviews of some of the major features.
Tracking changes to your MySQL configuration
Maintaining server configuration and tracking changes to it plays a vital role in the maintenance of MySQL servers. DBAs may be responsible for hundreds of servers and keeping an eye on the configuration settings for all of them could be difficult to say the least. With MONyog 4.0 now you can compare global MySQL configuration of multiple servers side-by-side, with all changes highlighted so that differences are visually discernible at a glance. Wondering why server A isn’t performing as well as server B when they share the exact same load? The answer could lie in the configuration files!
What’s more is that MONyog now lets you track changes to the global configuration of your MySQL server no matter if the configuration parameters were specified in my.ini/my.cnf, are server defaults or if somebody with SUPER privilege has executed a SET GLOBAL statement.
Improved customization framework
Many MONyog users have long since complained that the customization feature was complex and hard to use: One had to be well-versed in object-oriented JavaScript concepts to even make a minor change. Moreover, unless you manually backed up your customizations, you’d loose all of them if you decided to upgrade! That was motivation enough for us to redesign the entire framework.
With MONyog 4.0 knowledge of writing basic JavaScript functions is more than sufficient to customize any Monitor/Advisor or roll out your very own set. A simple form-based interface with verbose instructions makes it very easy to define a new Monitor/Advisor.
We understand that not every user will make use of the 220+ Monitors/Advisors that we ship with MONyog. So why waste all the computing power? You can now disable a subset of the Monitors/Advisors and keep only the ones you use. If you change your mind later, you can always re-enable them again.
MONyog 4.0 provides a unique customization interface. Now you won’t have to worry about loosing your changes, making mistakes (you can always revert to the original!), or feel scared to experiment.
Improved stability and lots of minor features and bug fixes
As always, this release too contains a lot of bug fixes. MONyog has undergone severe stress testing and is monitoring 200 without issues in our test environment now. From this release we have reduced the number of Linux binaries. In version 3.72 we introduced packages specific to glibc versions. It was necessary at that time to ensure stability on both older Linux distros (RHEL3-generation) as well as the most recent distros using glibc version 2.5. But we never liked it ourselves. It could confuse users – and sometimes did. Since then we have been researching and testing different packaging concepts so that a single binary will do the job on all distros. This release contains a single 32 and a single 64 binary – each with the option of a .tar.gz archive and a RPM installer.
MONyog customers can download the latest installers from Webyog’s Customer Portal.
We are very excited about this release, and hope that you will like it. We would love to hear from you.
Regards,
Team MONyog
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:6:"Webyog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:40;a:6:{s:4:"data";s:68:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:42:"Getting LuaSocket to Work with MySQL Proxy";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:35:"http://www.chriscalender.com/?p=126";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:35:"http://www.chriscalender.com/?p=126";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2045:"I’ve seen some interest in trying to get LuaSocket working with MySQL Proxy, and most of those interested have run into issues getting this set up, so I want to provide a brief walk-through on how to get this set-up and working properly.
Note I tested this on Windows, so use .so instead of .dll on *nix systems.
1. Download LuaSocket (contains lua/ and lib/ directories)
2. Copy contents of ‘lua/’ into the following directory:
C:\Program Files\MySQL\mysql-proxy-0.8.0\lib\mysql-proxy\lua
3. Copy contents of ‘lib/’ into the following directory:
C:\Program Files\MySQL\mysql-proxy-0.8.0\bin
4. In step #3, you should have copied a ’socket/’ and ‘mime/’ directories into bin/. Rename the directory named “socket” to “lua-socket”. This will get you past one set of errors.
5. Next, in the C:\Program Files\MySQL\mysql-proxy-0.8.0\bin, there is a file named lua51.dll. If you try to run it now, you’ll get an error saying “lua5.1.dll was not found”. So make a *copy* of “lua51.dll” and rename it “lua5.1.dll” (in the same directory).
6. Invoke your proxy script with the correct –lua-cpath and –lua-path options (below is correct relative to my basedir and with regards to where I copied the contents of the LuaSocket):
C:\Program Files\MySQL\mysql-proxy-0.8.0\bin>mysql-proxy.exe --proxy-lua-script=show-vars.lua --lua-cpath=!\\lua-?.dll --lua-path=!\\..\\lib\\mysql-proxy\\lua\\?.lua;!\\..\\lib\\mysql-proxy\\lua\\socket\\?.lua
Proxy should start up fine with the script, it works, and you can access the socket functions within your own lua script.
For instance, my proxy basedir, which is also where my script is located, is:
C:\Program Files\MySQL\mysql-proxy-0.8.0\
show-vars.lua is the name of my lua script.
In it, I have the following 2 lines:
socket = require("socket")
print(socket._VERSION)
And when I start up the script, and connect to my instance, it outputs the following:
LuaSocket 2.0.1";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Sat, 21 Aug 2010 02:21:06 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:6:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:9:"LuaSocket";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:11:"mysql proxy";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:5:"proxy";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:11:"proxy 0.8.0";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:6:"socket";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2504:"I’ve seen some interest in trying to get LuaSocket working with MySQL Proxy, and most of those interested have run into issues getting this set up, so I want to provide a brief walk-through on how to get this set-up and working properly.
Note I tested this on Windows, so use .so instead of .dll on *nix systems.
1. Download LuaSocket (contains lua/ and lib/ directories)
2. Copy contents of ‘lua/’ into the following directory:
C:\Program Files\MySQL\mysql-proxy-0.8.0\lib\mysql-proxy\lua
3. Copy contents of ‘lib/’ into the following directory:
C:\Program Files\MySQL\mysql-proxy-0.8.0\bin
4. In step #3, you should have copied a ’socket/’ and ‘mime/’ directories into bin/. Rename the directory named “socket” to “lua-socket”. This will get you past one set of errors.
5. Next, in the C:\Program Files\MySQL\mysql-proxy-0.8.0\bin, there is a file named lua51.dll. If you try to run it now, you’ll get an error saying “lua5.1.dll was not found”. So make a *copy* of “lua51.dll” and rename it “lua5.1.dll” (in the same directory).
6. Invoke your proxy script with the correct –lua-cpath and –lua-path options (below is correct relative to my basedir and with regards to where I copied the contents of the LuaSocket):
C:\Program Files\MySQL\mysql-proxy-0.8.0\bin>mysql-proxy.exe --proxy-lua-script=show-vars.lua --lua-cpath=!\\lua-?.dll --lua-path=!\\..\\lib\\mysql-proxy\\lua\\?.lua;!\\..\\lib\\mysql-proxy\\lua\\socket\\?.lua
Proxy should start up fine with the script, it works, and you can access the socket functions within your own lua script.
For instance, my proxy basedir, which is also where my script is located, is:
C:\Program Files\MySQL\mysql-proxy-0.8.0\
show-vars.lua is the name of my lua script.
In it, I have the following 2 lines:
socket = require("socket")
print(socket._VERSION)
And when I start up the script, and connect to my instance, it outputs the following:
LuaSocket 2.0.1
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:14:"Chris Calender";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:41;a:6:{s:4:"data";s:93:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:35:"Live video stream from OpenSQL Camp";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.lenzg.net/archives/308-guid.html";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:74:"http://www.lenzg.net/archives/308-Live-video-stream-from-OpenSQL-Camp.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:821:"Greetings from Sankt Augustin, Germany! I've arrived by train today and just returned from the FrOSCon venue, which will start tomorrow. The organizers are still busy with the preparations, but things already seem to be in good shape.It was a mild and sunny evening today. Hopefully it will be the same tomorrow again, so we can enjoy a relaxed BBQ outside! The social event at FrOSCon is always a nice opportunity to meet and talk with fellow open source enthusiasts, users and developers.And finally some good news for those of you who can't make it to FrOSCon this year: there will be live video streams from selected lecture rooms! So you will be able to attend the OpenSQL Camp sessions virtually - just head over to http://live.froscon.org/ and select room "HS6". It'll be interesting to see how this will work out.";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Fri, 20 Aug 2010 19:13:33 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:11:{i:0;a:5:{s:4:"data";s:5:"Linux";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:13:"collaborating";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:9:"community";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:10:"conference";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:9:"databases";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:5:"linux";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:8;a:5:{s:4:"data";s:3:"OSS";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:9;a:5:{s:4:"data";s:9:"streaming";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:10;a:5:{s:4:"data";s:5:"video";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1166:"Greetings from Sankt Augustin, Germany! I've arrived by train today and just returned from the FrOSCon venue, which will start tomorrow. The organizers are still busy with the preparations, but things already seem to be in good shape.
It was a mild and sunny evening today. Hopefully it will be the same tomorrow again, so we can enjoy a relaxed BBQ outside! The social event at FrOSCon is always a nice opportunity to meet and talk with fellow open source enthusiasts, users and developers.
And finally some good news for those of you who can't make it to FrOSCon this year: there will be live video streams from selected lecture rooms! So you will be able to attend the OpenSQL Camp sessions virtually - just head over to http://live.froscon.org/ and select room "HS6". It'll be interesting to see how this will work out.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Lenz Grimmer";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:42;a:6:{s:4:"data";s:53:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:46:"SHOW CREATE TABLE using table protobuf message";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:40:"http://www.flamingspork.com/blog/?p=2121";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:91:"http://www.flamingspork.com/blog/2010/08/20/show-create-table-using-table-protobuf-message/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1351:"… and really testing the replication code path for CREATE TABLE.
So, for a very long time now, Drizzle has been using a protobuf based structure to describe the structure of tables. The idea was to be able to have engines rather painlessly generate this structure themselves (which several now do). A secondary idea was to use this structure itself for CREATE TABLE (in progress, and embedded_innodb does in fact does only use the table message for its CREATE TABLE codepath). The third idea was to just put the table protobuf message into the replication stream instead of the CREATE TABLE statement (i.e. a SQL string). This means that you could (easily) write a replicator to a DBMS with different SQL syntax, or to a system that doesn’t speak SQL at all.
The final step, to reduce duplicated code functionality, would be to have the code that does SHOW CREATE TABLE to use a common bit of code for turning a table protobuf message back into a SQL string.
We now have that.
Just now, my branch to replace the old SHOW CREATE TABLE code (that was using TableShare and friends) with the statement_transform code (that we also use in converting a replication log to SQL commands) hit trunk.
Yay.
Share this on Facebook
Tweet This!
Share this on del.icio.us
Digg this!
Post on Google Buzz
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Fri, 20 Aug 2010 03:37:45 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:3:{i:0;a:5:{s:4:"data";s:7:"drizzle";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:8:"protobuf";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:17:"SHOW CREATE TABLE";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2891:"… and really testing the replication code path for CREATE TABLE.
So, for a very long time now, Drizzle has been using a protobuf based structure to describe the structure of tables. The idea was to be able to have engines rather painlessly generate this structure themselves (which several now do). A secondary idea was to use this structure itself for CREATE TABLE (in progress, and embedded_innodb does in fact does only use the table message for its CREATE TABLE codepath). The third idea was to just put the table protobuf message into the replication stream instead of the CREATE TABLE statement (i.e. a SQL string). This means that you could (easily) write a replicator to a DBMS with different SQL syntax, or to a system that doesn’t speak SQL at all.
The final step, to reduce duplicated code functionality, would be to have the code that does SHOW CREATE TABLE to use a common bit of code for turning a table protobuf message back into a SQL string.
We now have that.
Just now, my branch to replace the old SHOW CREATE TABLE code (that was using TableShare and friends) with the statement_transform code (that we also use in converting a replication log to SQL commands) hit trunk.
Yay.
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Stewart Smith";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:43;a:6:{s:4:"data";s:63:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:24:"Announcing TokuDB v4.1.1";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:26:"http://tokutek.com/?p=1647";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:52:"http://tokutek.com/2010/08/announcing-tokudb-v4-1-1/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1068:"Tokutek is pleased to announce immediate availability of TokuDB for MySQL, version 4.1.1. It is ideally suited for delivering fast response times for complex / high-volume Web applications that must simultaneously store and query large volumes of rapidly arriving data:
Social Networking
Real-time clickstream analysis
Logfile Analysis
eCommerce Personalization
High-speed Webcrawling
TokuDB v4.1.1 replaces TokuDB v4.1.0 and is recommended for all users. (We found a bug in v4.1.0 and have withdrawn it from our website). The new version has all of v4.1.0′s new features, including support for SAVEPOINT and an even better Fast Loader. As always, this release uses our high-performance Fractal Tree™ indexing to provide a unique combination of capabilities:
10x-50x faster indexing for faster querying
Full support for ACID transactions
Short recovery time (seconds or minutes, not hours or days)
Immunity to database aging to eliminate performance degradation and maintenance headaches
5x-15x data compression for reduced disk use and lower storage costs
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 23:36:12 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:5:{i:0;a:5:{s:4:"data";s:8:"TokuView";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:12:"Announcement";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:23:"Fractal Tree™ indexes";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:6:"TokuDB";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1466:"Tokutek is pleased to announce immediate availability of TokuDB for MySQL, version 4.1.1. It is ideally suited for delivering fast response times for complex / high-volume Web applications that must simultaneously store and query large volumes of rapidly arriving data:
- Social Networking
- Real-time clickstream analysis
- Logfile Analysis
- eCommerce Personalization
- High-speed Webcrawling
TokuDB v4.1.1 replaces TokuDB v4.1.0 and is recommended for all users. (We found a bug in v4.1.0 and have withdrawn it from our website). The new version has all of v4.1.0′s new features, including support for SAVEPOINT and an even better Fast Loader. As always, this release uses our high-performance Fractal Tree™ indexing to provide a unique combination of capabilities:
- 10x-50x faster indexing for faster querying
- Full support for ACID transactions
- Short recovery time (seconds or minutes, not hours or days)
- Immunity to database aging to eliminate performance degradation and maintenance headaches
- 5x-15x data compression for reduced disk use and lower storage costs
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Tokuview Blog";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:44;a:6:{s:4:"data";s:78:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:14:"Securich 0.3.0";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:41:"http://mysqlpreacher.com/wordpress/?p=391";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:58:"http://mysqlpreacher.com/wordpress/2010/08/securich-0-3-0/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:2202:"Around these days last year I presented `securich` for the first time. It was at froscon 2009, barely knowing anybody, spending my 27th birthday in a hostel in Germany fixing some bugs before the actual presentation on a 10 inch netbook (my mac had some problems at the time but thats another story :)). I got a beating, verbally of course! Many of the people listening to the presentation were expecting something else since another presentation was supposed to be running at that time, some even started dozing off (encouraging? not really hehe) but after a few minutes people started getting interested and asking all kind of questions. “This awesome” I thought to myself, “questions are good, it means people are understanding and want to know more”, but the more they learnt the more they realised how young securich was as a tool, lacking fundamental features like reverse reconciling securich with mysql rather than the other way round, bugs cropping up (Giuseppe aka datacharmer made sure he found some on the fly :P) but oh well back then it was just four months old.
Now after a year and four months into it, Securich is still in Beta but MANY MANY more features were included, bugs fixed (others created) and it even made it through two sessions at the Oreilly MySQL Conference and Expo 2010 and two mysql university sessions.
The end points here are:
1. Share your knowledge and code with the community, we learn, you learn!
2. When you fall to the ground, climb back up and get on your feet again. Start running!
3. Securich is now at version 0.3.0 (10th official release), help yourself at google code your feedback is greatly appreciated!
Most of the new release is bug fixes but some of the features added in this version are:
1. Any user can now install securich not just mysql root
2. You can now grant privileges on the mysql db (by default still disabled as it runs in “strict” mode which needs to be changed to “lenient” manually)
3. The installation script now keeps the securich package intact for further many installations
4. Better error reporting (for debugging purposes)
5. Grants on tables using regexp are now case sensitive";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 21:59:19 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:8:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:13:"Uncategorized";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:10:"conference";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:7:"froscon";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:6:"grants";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:5:"roles";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:8:"securich";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:8:"security";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2698:"Around these days last year I presented `securich` for the first time. It was at froscon 2009, barely knowing anybody, spending my 27th birthday in a hostel in Germany fixing some bugs before the actual presentation on a 10 inch netbook (my mac had some problems at the time but thats another story :)). I got a beating, verbally of course! Many of the people listening to the presentation were expecting something else since another presentation was supposed to be running at that time, some even started dozing off (encouraging? not really hehe) but after a few minutes people started getting interested and asking all kind of questions. “This awesome” I thought to myself, “questions are good, it means people are understanding and want to know more”, but the more they learnt the more they realised how young securich was as a tool, lacking fundamental features like reverse reconciling securich with mysql rather than the other way round, bugs cropping up (Giuseppe aka datacharmer made sure he found some on the fly :P) but oh well back then it was just four months old.
Now after a year and four months into it, Securich is still in Beta but MANY MANY more features were included, bugs fixed (others created) and it even made it through two sessions at the Oreilly MySQL Conference and Expo 2010 and two mysql university sessions.
The end points here are:
1. Share your knowledge and code with the community, we learn, you learn!
2. When you fall to the ground, climb back up and get on your feet again. Start running!
3. Securich is now at version 0.3.0 (10th official release), help yourself at google code your feedback is greatly appreciated!
Most of the new release is bug fixes but some of the features added in this version are:
1. Any user can now install securich not just mysql root
2. You can now grant privileges on the mysql db (by default still disabled as it runs in “strict” mode which needs to be changed to “lenient” manually)
3. The installation script now keeps the securich package intact for further many installations
4. Better error reporting (for debugging purposes)
5. Grants on tables using regexp are now case sensitive
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Darren Cassar";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:45;a:6:{s:4:"data";s:43:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:47:"MySQL at Oracle Open World - San Francisco 2010";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:70:"tag:blogger.com,1999:blog-6480555434853676325.post-2385210783636588869";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:110:"http://feedproxy.google.com/~r/MysqlDba-AnOracleDbasJourney/~3/BdI3H158Iz0/mysql-at-oracle-open-world-san.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:400:"Everybody is very interested in understanding the Oracle message about MySQL. Oracle is responding with a "MySQL Sunday" and a number of excellent presentations throughout the week in San Francisco at Oracle Open World. My session on MySQL is:
Session S316920 - The Ultimate Bootstrap for MySQL on Windows
There are also a number of excellent old MySQLers and friends presenting like:
Sarah ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 13:55:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:1:{i:0;a:5:{s:4:"data";s:16:"MySQL - OOW 2010";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:722:"Everybody is very interested in understanding the Oracle message about MySQL. Oracle is responding with a "MySQL Sunday" and a number of excellent presentations throughout the week in San Francisco at Oracle Open World. My session on MySQL is:
Session S316920 - The Ultimate Bootstrap for MySQL on Windows
There are also a number of excellent old MySQLers and friends presenting like:
Sarah
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:15:"George Trujillo";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:46;a:6:{s:4:"data";s:48:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:84:"High availability for MySQL on Amazon EC2 – Part 4 – The instance restart script";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.mysqlperformanceblog.com/?p=3457";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:134:"http://www.mysqlperformanceblog.com/2010/08/19/high-availability-for-mysql-on-amazon-ec2-%E2%80%93-part-4-the-instance-restart-script/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:11571:"This post is the fourth of a series that started here.
From the previous of this series, we now have resources configured but instead of starting MySQL, Pacemaker invokes a script to start (or restart) the EC2 instance running MySQL. This blog post describes the instance restart script. Remember, I am more a DBA than a script writer so it might not be written in the most optimal way.
First, let's recap what's the script has to perform (the full script is given below).
Kill the MySQL EC2 instance if running
Make sure the MySQL EC2 instance is stopped
Prepare the user-data script for the new MySQL EC2 instance
Launch the new MySQL instance
Make sure it is running
Reconfigure local heartbeat
Broadcast the new MySQL instance IP to the application servers
Kill the MySQL EC2 instance
In order to kill the existing MySQL EC2 instance, we first have to identify it. This is done by:
PLAIN TEXT
CODE:
OLD_INSTANCE_ID=`ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $AMI_HA_MYSQL | egrep "running|pending" | tail -n 1 | cut -d'|' -f3`
by filtering on the AMI type of the instance. Since an instance can be listed at the "stopped" state, it is mandatory to filter for states "running" or "pending". Then the instance is terminated with:
PLAIN TEXT
CODE:
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
Make sure the MySQL EC2 instance is stopped
Terminating an EC2 instance is not instantaneous, we can confirm an instance is really stopped by monitoring its status and wait until it is actually "terminated". The code below is how the script performs this task.
PLAIN TEXT
CODE:
#wait until the old instance is terminated it takes a few seconds to stop
done="false"
while [ $done == "false" ]
do
status=`ec2-describe-instances -K $PK -C $CERT $OLD_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | grep -c terminated`
if [ "$status" -eq "1" ]; then
done="true"
else
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
sleep 5
fi
done
Prepare the user-data script for the new MySQL EC2 instance
The new MySQL instance will be running heartbeat. Since we cannot use neither Ethernet broadcast or multicast, we need to configure the new instance so that it communicates through unicast with its partner node in the cluster, the node on which the restart script is run. This configuration is achieved by providing a user-data script (see the hamysql.user-data below) which completes the heartbeat configuration of the new instance. The hamysql.user-data script only performs a search and replace operation on the /etc/ha.d/ha.cf file and then restart the heartbeat service. In order for this to work properly, we just have to put the IP of the current instance in the script like here:
PLAIN TEXT
CODE:
OUR_IP=`/sbin/ifconfig eth0 | grep 'inet addr' | cut -d':' -f2 | cut -d' ' -f1`
#Now, modify the user-data script, we need to put our IP address in
if [ "$OUR_IP" == "" ]
then
echo "Error getting Our IP"
else
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $OUR_IP/g" $USER_DATA_SCRIPT
fi
Launch the new MySQL instance
Once things are ready, a new MySQL instance can be launched with:
PLAIN TEXT
CODE:
#Now we are ready to start a new one
INSTANCE_INFO=`ec2-run-instances -K $PK -C $CERT $AMI_HA_MYSQL -n 1 -g $HA_SECURITY_GROUP -f $USER_DATA_SCRIPT -t $INSTANCE_TYPE -z $INSTANCE_ZONE -k $INSTANCE_KEY | /usr/local/bin/filtre_instances.pl`
#wait until the new instance is running it take a few seconds to start
NEW_INSTANCE_ID=`echo $INSTANCE_INFO | cut -d'|' -f3`
Out of this operation, we retrieve the new instance "instance_id".
Make sure it is running
Since we know the "instance_id" of the new instance, checking if it is running is easy:
PLAIN TEXT
CODE:
done="false"
while [ $done == "false" ]
do
INSTANCE_INFO=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl`
status=`echo $INSTANCE_INFO | grep -c running`
if [ "$status" -eq "1" ]; then
done="true"
else
sleep 5
fi
done
Reconfigure local heartbeat
Now, Heartbeat, on the monitoring host, must be informed of the IP address of its new partner. In order to achieve this, a search and replace operation in the local ha.cf file followed of restart of Heartbeat is sufficient.
PLAIN TEXT
CODE:
#Set the IP in /etc/ha.d/ha.cf and ask heartbeat to reload its config
MYSQL_IP=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | cut -d'|' -f2`
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $MYSQL_IP/g" /etc/ha.d/ha.cf
/etc/init.d/heartbeat reload
Broadcast the new MySQL instance IP to the application servers
The final phase is to inform the application servers that the IP of the MySQL has changed. The best way to list those application servers is through a security group and, provided the appropriate ssh keys have been exchanged, this code will push the IP update.
PLAIN TEXT
CODE:
TMPFILE=`mktemp`
ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $CLIENT_SECURITY_GROUP> $TMPFILE
while read line
do
IP=`echo $line | cut -d'|' -f2`
ssh -i /usr/local/bin/update_mysql ubuntu@$IP sudo ./updated_xinetd.sh $MYSQL_IP
done <$TMPFILE
rm $TMPFILE
The full script:
PLAIN TEXT
CODE:
#!/bin/bash
HA_SECURITY_GROUP=testyves
CLIENT_SECURITY_GROUP=hamysql-client
CLIENT_SCRIPT=/usr/local/bin/update_client.sh
AMI_HA_MYSQL=ami-84a74fed
EBS_DATA_VOL=vol-aefawf
USER_DATA_SCRIPT=/usr/local/bin/hamysql.user-data
PK=/usr/local/bin/pk-FNMBRRABFRKVICBDZ4IOOSF7YROYZRZW.pem
CERT=/usr/local/bin/cert-FNMBRRABFRKVICBDZ4IOOSF7YROYZRZW.pem
INSTANCE_TYPE=m1.small
INSTANCE_ZONE=us-east-1c
INSTANCE_KEY=yves-key
OUR_IP=`/sbin/ifconfig eth0 | grep 'inet addr' | cut -d':' -f2 | cut -d' ' -f1`
#Now, modify the user-data script, we need to put our IP address in
if [ "$OUR_IP" == "" ]
then
echo "Error getting Our IP"
else
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $OUR_IP/g" $USER_DATA_SCRIPT
fi
while [ 1 ]; do
#First thing to do, terminate the other instance ID
OLD_INSTANCE_ID=`ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $AMI_HA_MYSQL | egrep "running|pending" | tail -n 1 | cut -d'|' -f3`
if [ "$OLD_INSTANCE_ID" == "" ]
then
#no running instance
:
else
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
#wait until the old instance is terminated it takes a few seconds to stop
done="false"
while [ $done == "false" ]
do
status=`ec2-describe-instances -K $PK -C $CERT $OLD_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | grep -c terminated`
if [ "$status" -eq "1" ]; then
done="true"
else
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
sleep 5
fi
done
fi
#Now we are ready to start a new one
INSTANCE_INFO=`ec2-run-instances -K $PK -C $CERT $AMI_HA_MYSQL -n 1 -g $HA_SECURITY_GROUP -f $USER_DATA_SCRIPT -t $INSTANCE_TYPE -z $INSTANCE_ZONE -k $INSTANCE_KEY | /usr/local/bin/filtre_instances.pl`
#wait until the new instance is running it take a few seconds to start
NEW_INSTANCE_ID=`echo $INSTANCE_INFO | cut -d'|' -f3`
if [ "$NEW_INSTANCE_ID" == "" ]
then
echo "Error creating the new instance"
else
done="false"
while [ $done == "false" ]
do
INSTANCE_INFO=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl`
status=`echo $INSTANCE_INFO | grep -c running`
if [ "$status" -eq "1" ]; then
done="true"
else
sleep 5
fi
done
#Set the IP in /etc/ha.d/ha.cf and ask heartbeat to reload its config
MYSQL_IP=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | cut -d'|' -f2`
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $MYSQL_IP/g" /etc/ha.d/ha.cf
TMPFILE=`mktemp`
ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $CLIENT_SECURITY_GROUP> $TMPFILE
while read line
do
IP=`echo $line | cut -d'|' -f2`
ssh -i /usr/local/bin/update_mysql ubuntu@$IP sudo ./updated_xinetd.sh $MYSQL_IP
done <$TMPFILE
rm $TMPFILE
/etc/init.d/heartbeat reload
fi
sleep 300 # 5 min before attempting again. Normally heartbeat should kill the script before
done
The hamysql.user-data script:
The script sets the IP of the monitor host in the heartbeat ha.cf configuration file and then, finishes up some missing configuration settings of the AMI.
PLAIN TEXT
CODE:
#!/bin/bash
sudo hostname hamysql
sudo perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 10.220.230.18/g" /etc/ha.d/ha.cf
# to eventually be added to the ebs image
sudo perl -pi -e 's/bind-address/#bind-address/g' /etc/mysql/my.cnf
sudo service mysql restart
sleep 5
/usr/bin/mysql -u root -proot -e "grant all on *.* to root@'%' identified by 'root'"
sudo /etc/init.d/heartbeat start
Entry posted by Yves Trudeau |
No comment
Add to: | | | | ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 12:54:21 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:2:{i:0;a:5:{s:4:"data";s:17:"High Availability";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:21098:"This post is the fourth of a series that started here.
From the previous of this series, we now have resources configured but instead of starting MySQL, Pacemaker invokes a script to start (or restart) the EC2 instance running MySQL. This blog post describes the instance restart script. Remember, I am more a DBA than a script writer so it might not be written in the most optimal way.
First, let's recap what's the script has to perform (the full script is given below).
- Kill the MySQL EC2 instance if running
- Make sure the MySQL EC2 instance is stopped
- Prepare the user-data script for the new MySQL EC2 instance
- Launch the new MySQL instance
- Make sure it is running
- Reconfigure local heartbeat
- Broadcast the new MySQL instance IP to the application servers
Kill the MySQL EC2 instance
In order to kill the existing MySQL EC2 instance, we first have to identify it. This is done by:
CODE:
-
OLD_INSTANCE_ID=`ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $AMI_HA_MYSQL | egrep "running|pending" | tail -n 1 | cut -d'|' -f3`
by filtering on the AMI type of the instance. Since an instance can be listed at the "stopped" state, it is mandatory to filter for states "running" or "pending". Then the instance is terminated with:
CODE:
-
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
Make sure the MySQL EC2 instance is stopped
Terminating an EC2 instance is not instantaneous, we can confirm an instance is really stopped by monitoring its status and wait until it is actually "terminated". The code below is how the script performs this task.
CODE:
-
#wait until the old instance is terminated it takes a few seconds to stop
-
done="false"
-
while [ $done == "false" ]
-
do
-
status=`ec2-describe-instances -K $PK -C $CERT $OLD_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | grep -c terminated`
-
if [ "$status" -eq "1" ]; then
-
done="true"
-
else
-
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
-
sleep 5
-
fi
-
done
Prepare the user-data script for the new MySQL EC2 instance
The new MySQL instance will be running heartbeat. Since we cannot use neither Ethernet broadcast or multicast, we need to configure the new instance so that it communicates through unicast with its partner node in the cluster, the node on which the restart script is run. This configuration is achieved by providing a user-data script (see the hamysql.user-data below) which completes the heartbeat configuration of the new instance. The hamysql.user-data script only performs a search and replace operation on the /etc/ha.d/ha.cf file and then restart the heartbeat service. In order for this to work properly, we just have to put the IP of the current instance in the script like here:
CODE:
-
OUR_IP=`/sbin/ifconfig eth0 | grep 'inet addr' | cut -d':' -f2 | cut -d' ' -f1`
-
#Now, modify the user-data script, we need to put our IP address in
-
if [ "$OUR_IP" == "" ]
-
then
-
echo "Error getting Our IP"
-
else
-
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $OUR_IP/g" $USER_DATA_SCRIPT
-
fi
Launch the new MySQL instance
Once things are ready, a new MySQL instance can be launched with:
CODE:
-
#Now we are ready to start a new one
-
INSTANCE_INFO=`ec2-run-instances -K $PK -C $CERT $AMI_HA_MYSQL -n 1 -g $HA_SECURITY_GROUP -f $USER_DATA_SCRIPT -t $INSTANCE_TYPE -z $INSTANCE_ZONE -k $INSTANCE_KEY | /usr/local/bin/filtre_instances.pl`
-
-
#wait until the new instance is running it take a few seconds to start
-
NEW_INSTANCE_ID=`echo $INSTANCE_INFO | cut -d'|' -f3`
Out of this operation, we retrieve the new instance "instance_id".
Make sure it is running
Since we know the "instance_id" of the new instance, checking if it is running is easy:
CODE:
-
done="false"
-
while [ $done == "false" ]
-
do
-
INSTANCE_INFO=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl`
-
status=`echo $INSTANCE_INFO | grep -c running`
-
if [ "$status" -eq "1" ]; then
-
done="true"
-
else
-
sleep 5
-
fi
-
done
Reconfigure local heartbeat
Now, Heartbeat, on the monitoring host, must be informed of the IP address of its new partner. In order to achieve this, a search and replace operation in the local ha.cf file followed of restart of Heartbeat is sufficient.
CODE:
-
#Set the IP in /etc/ha.d/ha.cf and ask heartbeat to reload its config
-
MYSQL_IP=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | cut -d'|' -f2`
-
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $MYSQL_IP/g" /etc/ha.d/ha.cf
-
/etc/init.d/heartbeat reload
Broadcast the new MySQL instance IP to the application servers
The final phase is to inform the application servers that the IP of the MySQL has changed. The best way to list those application servers is through a security group and, provided the appropriate ssh keys have been exchanged, this code will push the IP update.
CODE:
-
TMPFILE=`mktemp`
-
ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $CLIENT_SECURITY_GROUP> $TMPFILE
-
-
while read line
-
do
-
IP=`echo $line | cut -d'|' -f2`
-
ssh -i /usr/local/bin/update_mysql ubuntu@$IP sudo ./updated_xinetd.sh $MYSQL_IP
-
done <$TMPFILE
-
-
rm $TMPFILE
The full script:
CODE:
-
#!/bin/bash
-
HA_SECURITY_GROUP=testyves
-
CLIENT_SECURITY_GROUP=hamysql-client
-
CLIENT_SCRIPT=/usr/local/bin/update_client.sh
-
AMI_HA_MYSQL=ami-84a74fed
-
EBS_DATA_VOL=vol-aefawf
-
USER_DATA_SCRIPT=/usr/local/bin/hamysql.user-data
-
PK=/usr/local/bin/pk-FNMBRRABFRKVICBDZ4IOOSF7YROYZRZW.pem
-
CERT=/usr/local/bin/cert-FNMBRRABFRKVICBDZ4IOOSF7YROYZRZW.pem
-
INSTANCE_TYPE=m1.small
-
INSTANCE_ZONE=us-east-1c
-
INSTANCE_KEY=yves-key
-
-
OUR_IP=`/sbin/ifconfig eth0 | grep 'inet addr' | cut -d':' -f2 | cut -d' ' -f1`
-
#Now, modify the user-data script, we need to put our IP address in
-
if [ "$OUR_IP" == "" ]
-
then
-
echo "Error getting Our IP"
-
else
-
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $OUR_IP/g" $USER_DATA_SCRIPT
-
fi
-
-
while [ 1 ]; do
-
-
#First thing to do, terminate the other instance ID
-
OLD_INSTANCE_ID=`ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $AMI_HA_MYSQL | egrep "running|pending" | tail -n 1 | cut -d'|' -f3`
-
-
if [ "$OLD_INSTANCE_ID" == "" ]
-
then
-
#no running instance
-
:
-
else
-
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
-
-
#wait until the old instance is terminated it takes a few seconds to stop
-
done="false"
-
while [ $done == "false" ]
-
do
-
status=`ec2-describe-instances -K $PK -C $CERT $OLD_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | grep -c terminated`
-
if [ "$status" -eq "1" ]; then
-
done="true"
-
else
-
ec2-terminate-instances -K $PK -C $CERT $OLD_INSTANCE_ID> /dev/null
-
sleep 5
-
fi
-
done
-
fi
-
-
#Now we are ready to start a new one
-
INSTANCE_INFO=`ec2-run-instances -K $PK -C $CERT $AMI_HA_MYSQL -n 1 -g $HA_SECURITY_GROUP -f $USER_DATA_SCRIPT -t $INSTANCE_TYPE -z $INSTANCE_ZONE -k $INSTANCE_KEY | /usr/local/bin/filtre_instances.pl`
-
-
#wait until the new instance is running it take a few seconds to start
-
NEW_INSTANCE_ID=`echo $INSTANCE_INFO | cut -d'|' -f3`
-
-
if [ "$NEW_INSTANCE_ID" == "" ]
-
then
-
echo "Error creating the new instance"
-
else
-
-
done="false"
-
while [ $done == "false" ]
-
do
-
INSTANCE_INFO=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl`
-
status=`echo $INSTANCE_INFO | grep -c running`
-
if [ "$status" -eq "1" ]; then
-
done="true"
-
else
-
sleep 5
-
fi
-
done
-
-
#Set the IP in /etc/ha.d/ha.cf and ask heartbeat to reload its config
-
MYSQL_IP=`ec2-describe-instances -K $PK -C $CERT $NEW_INSTANCE_ID | /usr/local/bin/filtre_instances.pl | cut -d'|' -f2`
-
perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 $MYSQL_IP/g" /etc/ha.d/ha.cf
-
-
TMPFILE=`mktemp`
-
ec2-describe-instances -K $PK -C $CERT | /usr/local/bin/filtre_instances.pl | grep $CLIENT_SECURITY_GROUP> $TMPFILE
-
-
while read line
-
do
-
IP=`echo $line | cut -d'|' -f2`
-
ssh -i /usr/local/bin/update_mysql ubuntu@$IP sudo ./updated_xinetd.sh $MYSQL_IP
-
done <$TMPFILE
-
-
rm $TMPFILE
-
-
/etc/init.d/heartbeat reload
-
fi
-
-
-
sleep 300 # 5 min before attempting again. Normally heartbeat should kill the script before
-
done
The hamysql.user-data script:
The script sets the IP of the monitor host in the heartbeat ha.cf configuration file and then, finishes up some missing configuration settings of the AMI.
CODE:
-
#!/bin/bash
-
sudo hostname hamysql
-
sudo perl -pi -e "s/ucast eth0 (\d+)(\.\d+){3}/ucast eth0 10.220.230.18/g" /etc/ha.d/ha.cf
-
-
# to eventually be added to the ebs image
-
sudo perl -pi -e 's/bind-address/#bind-address/g' /etc/mysql/my.cnf
-
sudo service mysql restart
-
sleep 5
-
/usr/bin/mysql -u root -proot -e "grant all on *.* to root@'%' identified by 'root'"
-
sudo /etc/init.d/heartbeat start
Entry posted by Yves Trudeau |
No comment
Add to: | | | |
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Yves Trudeau";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:47;a:6:{s:4:"data";s:58:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:42:"Percona talks at OpenSQL Camp this weekend";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.mysqlperformanceblog.com/?p=3507";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:90:"http://www.mysqlperformanceblog.com/2010/08/19/percona-talks-at-opensql-camp-this-weekend/";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:564:"Four Perconians (perconites?) will be at OpenSQL Camp in Sankt Augustin, Germany this weekend presenting talks on:
Recovery of Lost or Corrupted InnoDB Tables
Keep your MySQL backend online no matter what
XtraDB -- InnoDB on steroids
Xtrabackup for MySQL
If you would like to stop by and say hello, we are Aleksandr, Istvan, Morgan and Aurimas (pictures here).
If you can make the (approximate) location, but not the date, we also have training in Frankfurt in three weeks time.
Entry posted by Morgan Tocker |
One comment
Add to: | | | | ";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 10:18:11 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:4:{i:0;a:5:{s:4:"data";s:9:"community";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:11:"conferences";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:8:"training";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:2917:"Four Perconians (perconites?) will be at OpenSQL Camp in Sankt Augustin, Germany this weekend presenting talks on:
- Recovery of Lost or Corrupted InnoDB Tables
- Keep your MySQL backend online no matter what
- XtraDB -- InnoDB on steroids
- Xtrabackup for MySQL
If you would like to stop by and say hello, we are Aleksandr, Istvan, Morgan and Aurimas (pictures here).
If you can make the (approximate) location, but not the date, we also have training in Frankfurt in three weeks time.
Entry posted by Morgan Tocker |
One comment
Add to: | | | |
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:13:"Morgan Tocker";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:48;a:6:{s:4:"data";s:73:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:44:"Speaking at the DOAG Conference in Nuremberg";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.lenzg.net/archives/306-guid.html";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:83:"http://www.lenzg.net/archives/306-Speaking-at-the-DOAG-Conference-in-Nuremberg.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1703:"The DOAG ("Deutsche ORACLE Anwendergruppe e.V.") is the German association of users of Oracle products. In November 2010 (16th-18th), they will hold their annual Oracle Users Conference 2010, which will take place in Nuremberg, Germany.This event is quite an institution – it has been established more than 20 years ago already. This year, there will be more than 400 sessions by over 300 speakers. For the first time, this year's conference will also have a dedicated stream of sessions about MySQL and I'm happy to announce that I'll be there, too, giving an overview of the various MySQL Replication Technologies as well as a glimpse into a MySQL DBA's toolchest.Here's the full list of MySQL sessions, as of today:
Tuesday 2010-11-16
10:00-10:45: Keynote: Oracle strategy and plans for MySQL (Rich Mason)
11:00-11:45: MySQL Architekturen für Oracle DBA's (Oliver Sennhauser)
12:00-12:45: Stored procedures in MySQL (Konstantin Osipov)
13:00-13:45: MySQL Backup- und Recovery-Strategien (Matthias Jung)
15:00-15:45: MySQL Idiosyncrasies That Bite (Ronald Bradford)
16:00-16:45: Plugins: Die Vielfalt unterm Dach von MySQL (Erkan Yanar)
Wednesday 2010-11-17
09:00-09:45: MySQL professionell betreiben (Mario Beck)
10:00-10:45: PBXT: A Transactional Storage Engine for MySQL (Paul McCullagh)
12:00-12:45: Bereitstellung von Web-2.0-Anwendungen mit MySQL (Ralf Gebhardt)
13:00-13:45: MySQL Replikationstechnologien - eine Übersicht (Lenz Grimmer)
15:00-15:45: Blick in die Werkzeug-Kiste eines MySQL-DBAs (Lenz Grimmer)
16:00-16:45: MySQL Workbench für Einsteiger
Thursday 2010-11-18
13:00-13:45: Aufbau eines Data Warehouse in einer MySQL / Oracle DB Landschaft (Nicola Marangoni)
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 08:32:00 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:7:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:9:"community";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:10:"conference";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:6:"oracle";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:12:"presentation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:9:"usergroup";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:3541:"The DOAG ("Deutsche ORACLE Anwendergruppe e.V.") is the German association of users of Oracle products. In November 2010 (16th-18th), they will hold their annual Oracle Users Conference 2010, which will take place in Nuremberg, Germany.
This event is quite an institution – it has been established more than 20 years ago already. This year, there will be more than 400 sessions by over 300 speakers. For the first time, this year's conference will also have a dedicated stream of sessions about MySQL and I'm happy to announce that I'll be there, too, giving an overview of the various MySQL Replication Technologies as well as a glimpse into a MySQL DBA's toolchest.
Here's the full list of MySQL sessions, as of today:
Tuesday 2010-11-16
Wednesday 2010-11-17
Thursday 2010-11-18
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Lenz Grimmer";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}i:49;a:6:{s:4:"data";s:83:"
";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";s:5:"child";a:3:{s:0:"";a:6:{s:5:"title";a:1:{i:0;a:5:{s:4:"data";s:61:"OpenSQL Camp Europe: Some last-minute changes to the schedule";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"guid";a:1:{i:0;a:5:{s:4:"data";s:43:"http://www.lenzg.net/archives/305-guid.html";s:7:"attribs";a:1:{s:0:"";a:1:{s:11:"isPermaLink";s:5:"false";}}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:4:"link";a:1:{i:0;a:5:{s:4:"data";s:99:"http://www.lenzg.net/archives/305-OpenSQL-Camp-Europe-Some-last-minute-changes-to-the-schedule.html";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:11:"description";a:1:{i:0;a:5:{s:4:"data";s:1037:"In just two days the OpenSQL Camp Europe (hosted by the excellent FrOSCon) will kick off!We've had a few last-minute changes to the schedule, as some speakers could not make it due to visa or family issues. But we managed to keep all slots filled on both days and I think we've come up with a very nice program! In addition to the two sessions I already had, I've volunteered to take over Giuseppe's talk about Shooting from the Hip. MySQL at the Command Line.The most up-to-date version of the schedule is on the OpenSQL Camp Wiki, I expect the remaining changes to be reflected on the FrOSCon schedule during the day as well.I would like to express my special gratitude to the folks at Percona, who really did their utmost in order to find replacement speakers for their accepted talks.I look forward to this weekend. And if you're interested to attend: I still have a few vouchers for free admission to the conference left over. Please contact me directly if you'd like to get one – first come, first serve! See you on Saturday!";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:7:"pubDate";a:1:{i:0;a:5:{s:4:"data";s:31:"Thu, 19 Aug 2010 07:43:24 +0000";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}s:8:"category";a:9:{i:0;a:5:{s:4:"data";s:5:"MySQL";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:1;a:5:{s:4:"data";s:13:"collaborating";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:2;a:5:{s:4:"data";s:9:"community";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:3;a:5:{s:4:"data";s:10:"conference";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:4;a:5:{s:4:"data";s:5:"event";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:5;a:5:{s:4:"data";s:7:"froscon";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:6;a:5:{s:4:"data";s:5:"mysql";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:7;a:5:{s:4:"data";s:11:"opensqlcamp";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}i:8;a:5:{s:4:"data";s:12:"presentation";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:40:"http://purl.org/rss/1.0/modules/content/";a:1:{s:7:"encoded";a:1:{i:0;a:5:{s:4:"data";s:1649:"In just two days the OpenSQL Camp Europe (hosted by the excellent FrOSCon) will kick off!
We've had a few last-minute changes to the schedule, as some speakers could not make it due to visa or family issues. But we managed to keep all slots filled on both days and I think we've come up with a very nice program! In addition to the two sessions I already had, I've volunteered to take over Giuseppe's talk about Shooting from the Hip. MySQL at the Command Line.
The most up-to-date version of the schedule is on the OpenSQL Camp Wiki, I expect the remaining changes to be reflected on the FrOSCon schedule during the day as well.
I would like to express my special gratitude to the folks at Percona, who really did their utmost in order to find replacement speakers for their accepted talks.
I look forward to this weekend. And if you're interested to attend: I still have a few vouchers for free admission to the conference left over. Please contact me directly if you'd like to get one – first come, first serve! See you on Saturday!
PlanetMySQL Voting:
Vote UP /
Vote DOWN";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}s:32:"http://purl.org/dc/elements/1.1/";a:1:{s:7:"creator";a:1:{i:0;a:5:{s:4:"data";s:12:"Lenz Grimmer";s:7:"attribs";a:0:{}s:8:"xml_base";s:0:"";s:17:"xml_base_explicit";b:0;s:8:"xml_lang";s:0:"";}}}}}}}}}}}}}}}}s:4:"type";i:128;s:7:"headers";a:7:{s:4:"date";s:29:"Fri, 27 Aug 2010 04:56:58 GMT";s:6:"server";s:22:"Apache/2.2.15 (Fedora)";s:13:"last-modified";s:29:"Fri, 27 Aug 2010 04:46:21 GMT";s:13:"accept-ranges";s:5:"bytes";s:14:"content-length";s:6:"349395";s:10:"connection";s:5:"close";s:12:"content-type";s:8:"text/xml";}s:5:"build";s:14:"20090627192103";}