Setting up Hadoop HDFS in Pseudodistributed Mode

April 19th, 2016 No comments

Well, new to the big data world.

Following Appendix A in the book Hadoop: The Definitive Guide, 4th Ed, just get it to work. I’m running Ubuntu 14.04.

1. Download and unpack the hadoop package, and set environment variables in your ~/.bashrc.

export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
export HADOOP_HOME=~/hadoop-2.5.2
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin

Verify with:

# hadoop version
Hadoop 2.5.2
Subversion https://git-wip-us.apache.org/repos/asf/hadoop.git -r cc72e9b000545b86b75a61f4835eb86d57bfafc0
Compiled by jenkins on 2014-11-14T23:45Z
Compiled with protoc 2.5.0
From source with checksum df7537a4faa4658983d397abf4514320
This command was run using /home/gonwan/hadoop-2.5.2/share/hadoop/common/hadoop-common-2.5.2.jar

The 2.5.2 distribution package is build in 64bit for *.so files, use the 2.4.1 package if you want 32bit ones.

2. Edit config files in $HADOOP_HOME/etc/hadoop:

<?xml version="1.0"?>
<!-- core-site.xml -->
<configuration>
  <property>
    <name>fs.defaultFS</name>
    <value>hdfs://localhost/</value>
  </property>
</configuration>
<?xml version="1.0"?>
<!-- hdfs-site.xml -->
<configuration>
  <property>
    <name>dfs.replication</name>
    <value>1</value>
  </property>
</configuration>

3. Config SSH:
Hadoop needs to start daemons on hosts of a cluster via SSH connection. A public key is generated to avoid password input.

# ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
# cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys

Verify with:

# ssh localhost

4. Format HDFS filesystem:

# hdfs namenode -format

5. Start HDFS:

# start-dfs.sh
Starting namenodes on [localhost]
localhost: starting namenode, logging to /home/gonwan/hadoop-2.5.2/logs/hadoop-gonwan-namenode-gonwan-mate17.out
localhost: starting datanode, logging to /home/gonwan/hadoop-2.5.2/logs/hadoop-gonwan-datanode-gonwan-mate17.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /home/gonwan/hadoop-2.5.2/logs/hadoop-gonwan-secondarynamenode-gonwan-mate17.out

Verify running with jps command:

# jps
2535 NameNode
2643 DataNode
2931 Jps
2828 SecondaryNameNode

6. Some tests:

# hadoop fs -ls /
# hadoop fs -mkdir /test
# hadoop fs -put ~/.bashrc /
# hadoop fs -ls /
Found 2 items
-rw-r--r--   1 gonwan supergroup        215 2016-04-19 16:07 /.bashrc
drwxr-xr-x   - gonwan supergroup          0 2016-04-19 16:06 /test

7. Stop HDFS:

# stop-dfs.sh

8. If there is an error like:

Error: JAVA_HOME is not set and could not be found.

Just edit $HADOOP_HOME/etc/hadoop/hadoop-env.sh and export JAVA_HOME explicitly again here. It does happen under Debian. Not knowing why the environment variable is not passed over SSH.

9. You can also set HADOOP_CONF_DIR to use a separate config directory for convenience. But make sure you have the whole directory copied from the Hadoop package. Otherwise, nasty errors may occur.

Categories: Java Tags: ,

Enabling Fail2ban

February 14th, 2016 No comments

Hundreds lines of log in wordpress show that, attackers are just trying passwords via xmlrpc.php. Add protection using the WP fail2ban plugin, inspired by the post here.

Categories: Tools Tags: ,

Coroutines in C++/Boost

January 19th, 2016 No comments

Starting with 1.56, boost/asio provides asio::spawn() to work with coroutines. Just paste the sample code here, with minor modifications:

#include <boost/asio.hpp>
#include <boost/asio/spawn.hpp>
#include <boost/bind.hpp>
#include <boost/shared_ptr.hpp>
#include <boost/enable_shared_from_this.hpp>
#include <iostream>
using namespace std;
using boost::asio::ip::tcp;


class session: public boost::enable_shared_from_this<session>
{
public:
    explicit session(boost::asio::io_service &io_service)
        : socket_(io_service), timer_(io_service), strand_(io_service)
    {
    }
    tcp::socket &socket()
    {
        return socket_;
    }
    void go()
    {
        boost::asio::spawn(strand_, boost::bind(&session::echo, shared_from_this(), _1));
        boost::asio::spawn(strand_, boost::bind(&session::timeout, shared_from_this(), _1));
    }
private:
    void echo(boost::asio::yield_context yield)
    {
        try {
            char data[128];
            while (true) {
                timer_.expires_from_now(boost::posix_time::seconds(10));
                size_t n = socket_.async_read_some(boost::asio::buffer(data), yield);
                boost::asio::async_write(socket_, boost::asio::buffer(data, n), yield);
            }
        } catch (exception &) {
            socket_.close();
            timer_.cancel();
        }
    }
    void timeout(boost::asio::yield_context yield)
    {
        while (socket_.is_open()) {
            boost::system::error_code ignored_ec;
            timer_.async_wait(yield[ignored_ec]);
            if (timer_.expires_from_now() <= boost::posix_time::seconds(0)) {
                socket_.close();
            }
        }
    }
    tcp::socket socket_;
    boost::asio::deadline_timer timer_;
    boost::asio::io_service::strand strand_;
};

void do_accept(boost::asio::io_service &io_service, unsigned short port, boost::asio::yield_context yield)
{
    tcp::acceptor acceptor(io_service, tcp::endpoint(tcp::v4(), port));
    while (true) {
        boost::system::error_code ec;
        boost::shared_ptr<session> new_session(new session(io_service));
        acceptor.async_accept(new_session->socket(), yield[ec]);
        if (!ec) {
            new_session->go();
        }
    }
}

int main()
{
    try {
        boost::asio::io_service io_service;
        boost::asio::spawn(io_service, boost::bind(do_accept, boost::ref(io_service), 2222, _1));
        io_service.run();
    } catch (exception &e) {
        cerr << "Exception: " << e.what() << endl;
    }
    return 0;
}

The Python in my previous article can be used to work with the code above. I also tried to write a TCP server with only boost::coroutines classes. select() is used, since I want the code to be platform independent. NOTE: with coroutines, we have only _one_ thread.

#ifdef _WIN32
#include <winsock2.h>
#include <ws2tcpip.h>
#include <windows.h>
#pragma comment(lib, "ws2_32.lib")
#pragma warning(disable: 4996)
#define sock_send(s, str, len)      send(s, str, len, 0)
#define sock_close(s)               closesocket(s)
#else
#include <arpa/inet.h>
#include <sys/types.h>
#include <sys/socket.h>
#define sock_send(s, str, len)      send(s, str, len, MSG_NOSIGNAL)
#define sock_close(s)               close(s)
#endif
#include <cerrno>
#include <cstdio>
#include <cstring>
#include <iostream>
#include <list>
#include <boost/bind.hpp>
#include <boost/coroutine/all.hpp>
#include <boost/shared_ptr.hpp>
using namespace std;


#ifdef _WIN32
struct Win32SocketWrapper
{
    Win32SocketWrapper()
    {
        WSADATA wsaData;
        WSAStartup(0x0202, &wsaData);
    }
    ~Win32SocketWrapper()
    {
        WSACleanup();
    }
} g_win32_socket_wrapper;
#endif


class session
{
    typedef boost::coroutines::symmetric_coroutine<void> coro_t;
public:
    explicit session(int sock)
        : socket_(sock)
    {
        echo_coro_ = coro_t::call_type(boost::bind(&session::echo, this, _1));
    }
    int socket()
    {
        return socket_;
    }
    void go()
    {
        echo_coro_();
    }
    void echo(coro_t::yield_type &yield)
    {
        int rc;
        char buffer[128];
        while (true) {
            memset(buffer, 0, sizeof(buffer));
            yield(); rc = recv(socket_, buffer, sizeof(buffer), 0);
            if (rc == 0 || rc == -1) { /* close or error */
                printf("socket[%d] closed, rc=%d..\n", socket_, rc);
                sock_close(socket_);
                socket_ = -1;
                /* do not release here, or the whole coroutine context will be invalid.. */
                break;
            } else {
                sock_send(socket_, buffer, rc);
            }
        }
    }
private:
    int socket_;
    coro_t::call_type echo_coro_;
};

void event_loop(int server_sock)
{
    list<boost::shared_ptr<session> > session_list;
    int rc, maxfd, client_sock;
    fd_set rdset;
    struct sockaddr_in client_addr;
    size_t addr_size = sizeof(struct sockaddr_in);

    while (true) {
        FD_ZERO(&rdset);
        FD_SET(server_sock, &rdset);
        maxfd = server_sock;
        list<boost::shared_ptr<session> >::iterator it = session_list.begin();
        while (it != session_list.end()) {
            if ((*it)->socket() == -1) {
                session_list.erase(it++);
            } else {
                FD_SET((*it)->socket(), &rdset);
                if (maxfd < (*it)->socket()) {
                    maxfd = (*it)->socket();
                }
                ++it;
            }
        }
        /* max fd value plus 1 */
        rc = select(maxfd+1, &rdset, 0, 0, NULL);
        if (rc == -1) {
            continue;
        } else {
            if (FD_ISSET(server_sock, &rdset)) {
                client_sock = (int)accept(server_sock, (struct sockaddr *)&client_addr, (socklen_t *)&addr_size);
                printf("socket[%d] accepted: %s:%d..\n", client_sock, inet_ntoa(client_addr.sin_addr), ntohs(client_addr.sin_port));
                boost::shared_ptr<session> new_session(new session(client_sock));
                new_session->go(); /* go first */
                session_list.push_back(new_session);
            }
            for (list<boost::shared_ptr<session> >::iterator it = session_list.begin(); it != session_list.end(); ++it) {
                if (FD_ISSET((*it)->socket(), &rdset)) {
                    (*it)->go();
                }
            }
        }
    }
}

int main() 
{
    int rc, server_sock;
    struct sockaddr_in server_addr;

    server_sock = (int)socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
    server_addr.sin_family = AF_INET;
    server_addr.sin_addr.s_addr = htonl(INADDR_ANY);
    server_addr.sin_port = htons(2222);
    rc = bind(server_sock, (struct sockaddr *)&server_addr, sizeof(struct sockaddr_in));
    if (rc < 0) {
        fprintf(stderr, "bind: %s.\n", strerror(errno));
        return -1;
    }
    listen(server_sock, 5);
    /* loop */
    event_loop(server_sock);
    sock_close(server_sock);
    return 0;
}
Categories: C/C++ Tags: , ,

Coroutines in Python

January 11th, 2016 No comments

Python 3.5 added native support for coroutines. Actually, there were several steps towards the current implementation. See Wikipedia, and it seems a bit messy to me:

  • Python 2.5 implements better support for coroutine-like functionality, based on extended generators (PEP 342).
  • Python 3.3 improves this ability, by supporting delegating to a subgenerator (PEP 380).
  • Python 3.4 introduces a comprehensive asynchronous I/O framework as standardized in PEP 3156, which includes coroutines that leverage subgenerator delegation.
  • Python 3.5 introduces explicit support for coroutines with async/await syntax (PEP 0492).

Before Python 2.5, there were only generators.

In Python 2.5, yield was refined to be an expression rather than a statement, which gave the possibility to implement a simple coroutine. But still a lot of work left for programmers to use it. For instance, a simple conroutine scheduler was required.

In Python 3.3, yield from was added to support subgenerators. Nothing to do with coroutines.

In Python 3.4, the Father of Python (Guido van Rossum) wrote a PEP himself to add an asyncio module to simplify coroutine usage in Python. An official scheduler was added. We can use @asyncio.coroutine to decorate a function. We can use yield from expressions to yield to a specific coroutine.

In Python 3.5, async/await syntax was added, borrowed from C#. The newest PEP made coroutines a native Python language feature, and clearly separated them from generators. A native coroutine now declares with async def syntax, and yield from is replaced with await expression. This removes generator/coroutine ambiguity. So in Python 3.5, coroutines used with asyncio may be implemented using the async def statement, or by using generators. Generator-based coroutines should be decorated with @asyncio.coroutine, although this is not strictly enforced. The decorator enables compatibility with async def coroutines, and also serves as documentation. See Python documents here.

The implementation can be found in this commit.

I wrote a echo server/client sample to try corutines. Server code first:

#!/usr/bin/python3
import asyncio

@asyncio.coroutine
def start_server():
    yield from asyncio.start_server(client_connected_handler, '127.0.0.1', 2222)

@asyncio.coroutine
def client_connected_handler(client_reader, client_writer):
    peer = client_writer.get_extra_info('peername')
    print('Connected..%s:%s' % (peer[0], peer[1]))
    while True:
        data = yield from client_reader.read(1024)
        if not data:
            print('Disconnected..%s:%s\n' % (peer[0], peer[1]))
            break
        print(data.decode(), end='')
        client_writer.write(data)

loop = asyncio.get_event_loop()
server = loop.run_until_complete(start_server())
try:
    loop.run_forever()
except KeyboardInterrupt:
    pass

server.close()
loop.run_until_complete(server.wait_closed())
loop.close()

Client code here, or you can simply use telnet command:

#!/usr/bin/python3
import asyncio

@asyncio.coroutine
def tcp_echo_client():
    reader, writer = yield from asyncio.open_connection('127.0.0.1', 2222)
    writer.write(b'first line\n')
    writer.write(b'second line\n')
    writer.write(b'third line\n')
    writer.write(b'EOF\n')
    print("Lines received..")
    while True:
        line = yield from reader.readline()
        if not line:
            break
        line = line.decode()
        print(line, end='')
        if line == 'EOF\n':
            break
    writer.close()

loop = asyncio.get_event_loop()
loop.run_until_complete(tcp_echo_client())
loop.close()

Server output:

Connected..127.0.0.1:27643
first line
second line
third line
EOF
Disconnected..127.0.0.1:27643

Client output:

Lines received..
first line
second line
third line
EOF

With Python 3.5 on Ubuntu 16.04, we can also use async/await:

#!/usr/bin/python3
import asyncio

async def start_server():
    await asyncio.start_server(client_connected_handler, '127.0.0.1', 2222)

async def client_connected_handler(client_reader, client_writer):
    peer = client_writer.get_extra_info('peername')
    print('Connected..%s:%s' % (peer[0], peer[1]))
    while True:
        data = await client_reader.read(1024)
        if not data:
            print('Disconnected..%s:%s\n' % (peer[0], peer[1]))
            break
        print(data.decode(), end='')
        client_writer.write(data)

loop = asyncio.get_event_loop()
server = loop.run_until_complete(start_server())
try:
    loop.run_forever()
except KeyboardInterrupt:
    pass

server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
Categories: Python Tags: ,

Basic Usage of Boost MultiIndex Containers

December 28th, 2015 Comments off

Just take a simple note here.
The Boost Multi-index Containers Library provides a class template named multi_index_container which enables the construction of containers maintaining one or more indices with different sorting and access semantics.

#include <string>
#include <ostream>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/tag.hpp>
#include <boost/lambda/lambda.hpp>
using namespace std;
using namespace boost::multi_index;


struct auth
{
    string m_name;
    string m_pass;
    auth(const string &name, const string &pass)
        : m_name(name), m_pass(pass) { }
    bool operator<(const auth &o) const { return m_name < o.m_name; }
    friend ostream &operator<<(ostream &os, const auth &a) { os << "(" << a.m_name << ", " << a.m_pass << ")"; return os; }
};

struct employee
{
    int m_id;
    auth m_auth;
    int m_hire;
    int m_resign;
    employee(int id, const string &name, const string &pass, int hire, int resign)
        : m_id(id), m_auth(name, pass), m_hire(hire), m_resign(resign) { }
    bool operator<(const employee &o) const { return m_id < o.m_id; }
    friend ostream &operator<<(ostream &os, const employee &e) { os << "(" << e.m_id << ", " << e.m_auth << ", " << e.m_hire << ", " << e.m_resign << ")"; return os; }
};


struct auth_t { };

struct change_resign {
    int m_resign;
    change_resign(int r) : m_resign(r) { }
    void operator()(employee &e) { e.m_resign = m_resign; }
};

typedef multi_index_container<
    employee,
    indexed_by<
        /* sort by employee::operator< */
        ordered_unique<identity<employee> >,
        /* sort by less<int> on m_hire */
        ordered_non_unique<member<employee, int, &employee::m_hire> >,
        /* sort by less<auth> on m_auth */
        ordered_non_unique<tag<auth_t>, member<employee, auth, &employee::m_auth> >
    >
> employee_set;

int main()
{
    employee_set es;
    es.insert(employee(1, "555", "555pass", 2012, 0));
    es.insert(employee(2, "444", "444pass", 2011, 0));
    es.insert(employee(3, "333", "333pass", 2013, 0));
    es.insert(employee(4, "222", "222pass", 2015, 0));
    es.insert(employee(5, "555", "555pass", 2014, 0)); /* dup */
    typedef employee_set::nth_index<1>::type hire_index_t;
    typedef employee_set::index<auth_t>::type auth_index_t;
    cout << "Get a view to index #1 (m_hire).." << endl;
    hire_index_t &hire_index = es.get<1>();
    std::copy(hire_index.begin(), hire_index.end(), ostream_iterator<employee>(cout, "\n"));
    cout << "Get a view to index tag auth_t (m_auth).." << endl;
    const auth_index_t &auth_index = es.get<auth_t>();
    std::copy(auth_index.begin(), auth_index.end(), ostream_iterator<employee>(cout, "\n"));
    cout << "Find.." << endl;
    hire_index_t::iterator it = hire_index.find(2015);
#if 0
    employee t = *it;
    t.m_resign = 2048;
    hire_index.replace(it, t);
#else
    hire_index.modify_key(it, boost::lambda::_1=1111);
    int old_resign = it->m_resign;
    hire_index.modify(it, change_resign(2048), change_resign(old_resign));
#endif
    cout << (*it) << endl;
    cout << "Find all.." << endl;
    pair<auth_index_t::const_iterator, auth_index_t::const_iterator> pr = auth_index.equal_range(auth("555", ""));
    std::copy(pr.first, pr.second, ostream_iterator<employee>(cout, "\n"));
    return 0;
}

Output:

Get a view to index #1 (m_hire)..
(2, (444, 444pass), 2011, 0)
(1, (555, 555pass), 2012, 0)
(3, (333, 333pass), 2013, 0)
(5, (555, 555pass), 2014, 0)
(4, (222, 222pass), 2015, 0)
Get a view to index tag auth_t (m_auth)..
(4, (222, 222pass), 2015, 0)
(3, (333, 333pass), 2013, 0)
(2, (444, 444pass), 2011, 0)
(1, (555, 555pass), 2012, 0)
(5, (555, 555pass), 2014, 0)
Find..
(4, (222, 222pass), 1111, 2048)
Find all..
(1, (555, 555pass), 2012, 0)
(5, (555, 555pass), 2014, 0)

To use with pointer values, only limited change needed as highlighted:

#include <string>
#include <ostream>
#include <boost/multi_index_container.hpp>
#include <boost/multi_index/ordered_index.hpp>
#include <boost/multi_index/identity.hpp>
#include <boost/multi_index/member.hpp>
#include <boost/multi_index/tag.hpp>
#include <boost/lambda/lambda.hpp>
using namespace std;
using namespace boost::multi_index;


struct auth
{
    string m_name;
    string m_pass;
    auth(const string &name, const string &pass)
        : m_name(name), m_pass(pass) { }
    bool operator<(const auth &o) const { return m_name < o.m_name; }
    friend ostream &operator<<(ostream &os, const auth &a) { os << "(" << a.m_name << ", " << a.m_pass << ")"; return os; }
};

struct employee
{
    int m_id;
    auth m_auth;
    int m_hire;
    int m_resign;
    employee(int id, const string &name, const string &pass, int hire, int resign)
        : m_id(id), m_auth(name, pass), m_hire(hire), m_resign(resign) { }
    bool operator<(const employee &o) const { return m_id < o.m_id; }
    friend ostream &operator<<(ostream &os, const employee *e) { os << "(" << e->m_id << ", " << e->m_auth << ", " << e->m_hire << ", " << e->m_resign << ")"; return os; }
};


struct auth_t { };

struct change_resign {
    int m_resign;
    change_resign(int r) : m_resign(r) { }
    void operator()(employee *e) { e->m_resign = m_resign; }
};

typedef multi_index_container<
    employee *,
    indexed_by<
        /* sort by employee::operator< */
        ordered_unique<identity<employee> >,
        /* sort by less<int> on m_hire */
        ordered_non_unique<member<employee, int, &employee::m_hire> >,
        /* sort by less<auth> on m_auth */
        ordered_non_unique<tag<auth_t>, member<employee, auth, &employee::m_auth> >
    >
> employee_set;

int main()
{
    employee_set es;
    es.insert(new employee(1, "555", "555pass", 2012, 0));
    es.insert(new employee(2, "444", "444pass", 2011, 0));
    es.insert(new employee(3, "333", "333pass", 2013, 0));
    es.insert(new employee(4, "222", "222pass", 2015, 0));
    es.insert(new employee(5, "555", "555pass", 2014, 0)); /* dup */
    typedef employee_set::nth_index<1>::type hire_index_t;
    typedef employee_set::index<auth_t>::type auth_index_t;
    cout << "Get a view to index #1 (m_hire).." << endl;
    hire_index_t &hire_index = es.get<1>();
    std::copy(hire_index.begin(), hire_index.end(), ostream_iterator<employee *>(cout, "\n"));
    cout << "Get a view to index tag auth_t (m_auth).." << endl;
    const auth_index_t &auth_index = es.get<auth_t>();
    std::copy(auth_index.begin(), auth_index.end(), ostream_iterator<employee *>(cout, "\n"));
    cout << "Find.." << endl;
    hire_index_t::iterator it = hire_index.find(2015);
#if 0
    employee *t = *it;
    t->m_auth.m_name = "888";
    /* must use replace() to notify changes in indexed fields */
    hire_index.replace(it, t);
#else
    hire_index.modify_key(it, boost::lambda::_1=1111);
    int old_resign = (*it)->m_resign;
    hire_index.modify(it, change_resign(2048), change_resign(old_resign));
#endif
    cout << (*it) << endl;
    cout << "Find all.." << endl;
    pair<auth_index_t::const_iterator, auth_index_t::const_iterator> pr = auth_index.equal_range(auth("555", ""));
    std::copy(pr.first, pr.second, ostream_iterator<employee *>(cout, "\n"));
    /* clear */
    for (employee_set::const_iterator it = es.begin(); it != es.end(); ++it) {
        delete *it;
    }
    employee_set().swap(es);
    return 0;
}
Categories: C/C++ Tags: ,

Enabing Pretty Permalinks

December 28th, 2015 No comments

Well.. long time no see. Just have some time to optimize the site for better analysis.

According to the official tutorial:
1. Enable mod_rewrite in apache2.

# sudo a2enmod rewrite

2. Enable FollowSymLinks option, which is default.
3. Enable FileInfo directives. Edit /etc/apache2/sites-available/yoursite.com.conf, add:

  <Directory /home/yourname/yoursite/>
    Require all granted
    AllowOverride FileInfo
  </Directory>

4. Restart apache:

# sudo service apache2 restart

Updated Dec 31, 2015:
Enabling mod_rewrite rewrites all requests including the one used by mod_status. To disable this, add a rule to the .htaccess file.

RewriteCond %{REQUEST_URI} !=/server-status

Then, change its user & group attributes to prevent overwriting from apache.

Categories: Tools Tags:

3 New Plugins Installed

July 8th, 2015 No comments

Just added another personal site for one of my friends. So another chance to speed up loading for my own site, from mainland china >.<

1. Disable google fonts: Remove Google Fonts References

2. Use local avatar: WP User Avatar

3. Use local emoji: WP Local Emoji (installed, but not activated.)

Categories: Tools Tags: ,

Database Access Layer in C++

April 15th, 2015 No comments

We have JDBC in Java… and SOCI in C++… Well, it’s not so easy as it should be. To build with cmake:

# mkdir ../build
# cd ../build
# set ORACLE_HOME=C:\oraclexe\app\oracle\product\10.2.0\server
# cmake -G "Visual Studio 9 2008" -DMYSQL_INCLUDE_DIR="C:\Program Files\MySQL\MySQL Server 5.5\include" -DMYSQL_LIBRARIES="C:\Program Files\MySQL\MySQL Server 5.5\lib\libmysql.lib" ../soci-3.2.3

The documents seem outdated, many options do not work. Just managed to figure out from the *.cmake source files. You can also download the oracle instant client SDK, and re-arrange the directory structure for build.

Code snippet I extracted from its unit tests:

#include "soci.h"
#include "soci-mysql.h"
//#include "soci-oracle.h"
#include <ctime>
#include <string>
#include <iostream>
#include <sstream>
using namespace std;

int main()
{
    try {
        soci::session sql(soci::mysql, "host=192.168.1.101 db=mysql user=root password=111111");
        //soci::session sql(soci::oracle, "service=192.168.1.102/ORCL user=sys password=111111");
        soci::row v;
        /* comma operator is overloaded here.. */
        soci::statement st = (sql.prepare << "SELECT * FROM user", into(v));
        //soci::statement st = (sql.prepare << "SELECT * FROM SYS.USER$", into(v));
        st.execute(true);  /* with data exchange */
        unsigned int num_fields = v.size();
        cout << "fields: " << num_fields << endl;
        num_fields = (num_fields <= 9) ? num_fields : 9;
        unsigned long num_rows = (unsigned long)st.get_affected_rows();
        cout << "rows: " << num_rows << endl;
        for (size_t i = 0; i < num_fields; ++i) {
            const soci::column_properties &props = v.get_properties(i);
            cout << props.get_name() << '\t';
        }
        cout << endl;
        do {
            stringstream ss;
            for (size_t i = 0; i < num_fields; ++i) {
                if (v.get_indicator(i) == soci::i_null) {
                    ss << "NULL";
                    break;
                }
                const soci::column_properties &props = v.get_properties(i);
                switch (props.get_data_type()) {
                case soci::dt_string:
                    ss << v.get<string>(i);
                    break;
                case soci::dt_double:
                    ss << v.get<double>(i);
                    break;
                case soci::dt_integer:
                    ss << v.get<int>(i);
                    break;
                case soci::dt_long_long:
                    ss << v.get<long long>(i);
                    break;
                case soci::dt_unsigned_long_long:
                    ss << v.get<unsigned long long>(i);
                    break;
                case soci::dt_date:
                    tm dt = v.get<tm>(i);
                    ss << asctime(&dt);
                    break;
                }
                ss << '\t';
            }
            cout << ss.str() << endl;
        } while (st.fetch());
    } catch (soci::soci_error &e) {
        cerr << "Error: " << e.what() << endl;
    }
    return 0;
}

Updated Apr 20, 2015:

1. Under RHEL5/CentOS5, I got errors like:

./test_oracle: error while loading shared libraries: /home/gonwan/oracle11_64/lib/libnnz11.so: cannot restore segment prot after reloc: Permission denied

It’s due to SELinux security feature. Simply workaround it with:

# chcon -t texrel_shlib_t *.so*

2. Oracle uses oraociei11.dll or libociei.so for client data. They are both large files(110+MB), since they support multiple languages. Instead, you can use oraociicus11.dll(30+MB) or libociicus.so(10-MB). These files contain only English support.

Categories: C/C++ Tags: , ,

Mount Samba Share as a Non-root User

April 9th, 2015 No comments

I used to access windows share folders directly in nautilus, or mount them like:

# mount -t cifs -o username=<your_username>,password=<your_password> //<your_server>/<your_share> /mnt/<your_local>

The problem is, they can be accessed only by root. The solution is adding a simple uid option like:

# sudo mount -t cifs -o uid=<your_uid>,username=<your_username>,password=<your_password>,domain=<your_domain> //<your_server>/<your_share> /mnt/<your_local> -vvv

See: http://wiki.centos.org/TipsAndTricks/WindowsShares

Updated June 1, 2015:

You may encounter 121 error like:

mount error(121): Remote I/O error
Refer to the mount.cifs(8) manual page (e.g. man mount.cifs)

It’s a Windows side issue, set following registry value to 3. This value tells Windows to prioritize file sharing over reducing memory usage.

HKLM\SYSTEM\CurrentControlSet\Services\LanmanServer\Parameters\Size

Reboot (or just restart the “Server” service in services.msc). Your problem should now be solved.

See: https://boinst.wordpress.com/2012/03/20/mount-cifs-cannot-allocate-memory-mounting-windows-share/

Categories: Linux Tags:

MySQL with C++ and Python

March 26th, 2015 No comments

Backup here:

1. C++

/*
 * # gcc test_mysql.c -o test_mysql -lmysqlclient -pedantic -Wall -Wextra
 */
#include <stdio.h>
#include <mysql/mysql.h>

int main()
{
    const char *host = "192.168.1.101";
    const char *user = "root";
    const char *pass = "111111";
    const char *dbname = "mysql";
    MYSQL mysql;
    MYSQL_RES *mysql_res;
    MYSQL_FIELD *mysql_field;
    MYSQL_ROW mysql_row;
    unsigned int num_fields;
    unsigned long num_rows;
    int i, rc;

    mysql_init(&mysql);
    /* connect */
    if (!mysql_real_connect(&mysql, host, user, pass, dbname, 0, NULL, 0)) {
        fprintf(stderr, "ERROR: %s\n", mysql_error(&mysql));
        return -1;
    }
    /* query */
    rc = mysql_query(&mysql, "SELECT * FROM user");
    if (rc) {
        fprintf(stderr, "ERROR: %s\n", mysql_error(&mysql));
        mysql_close(&mysql);
        return -1;
    }
    /* display */
    mysql_res = mysql_store_result(&mysql);
    if (!mysql_res) {
        fprintf(stderr, "ERROR: %s\n", mysql_error(&mysql));
        mysql_close(&mysql);
        return -1;
    }
    num_fields = mysql_num_fields(mysql_res);
    printf("fields: %u\n", num_fields);
    num_rows = (unsigned long)mysql_num_rows(mysql_res);
    printf("rows: %lu\n", num_rows);
    if (num_rows != 0) {
        num_fields = (num_fields <= 9) ? num_fields : 9;
        for (i = 0; i < (int)num_fields; i++) {
            mysql_field = mysql_fetch_field_direct(mysql_res, i);
            printf("%s\t", mysql_field->name);
        }
        printf("\n");
        while ((mysql_row = mysql_fetch_row(mysql_res)) != NULL) {
            for (i = 0; i < (int)num_fields; i++) {
                printf("%s\t",  mysql_row[i]);
            }
            printf("\n");
        }
    }
    mysql_free_result(mysql_res);
    mysql_close(&mysql);
    return 0;
}

2. Python

#!/usr/bin/python
# sudo apt-get install python-mysql.connector
# sudo apt-get install python3-mysql.connector
from __future__ import print_function
import mysql.connector

cur = None
con = None

try:
    con = mysql.connector.connect(host='192.168.1.101', user='root', password='111111', database='mysql')
    cur = con.cursor()
    cur.execute("SELECT * FROM user")
    num_fields = len(cur.column_names)
    print('fields: %d' % num_fields)
    if num_fields > 9:
        num_fields = 9
    rows = cur.fetchall()  # fetchall() before get rowcount
    print('rows: %d' % cur.rowcount)
    for i in range(num_fields):
        print('%s\t' % (cur.column_names[i]), end='')
    print()
    for row in rows:
        for i in range(num_fields):
            print('%s\t' % (row[i]), end='')
        print()
except mysql.connector.Error as e:
    print(e)
finally:
    if cur:
        cur.close()
    if con:
        con.close()
Categories: Database Tags: