Archive

Posts Tagged ‘AWS’

Redis backup script

August 7, 2015 3 comments
#!/bin/bash 
## Fri, 07 Aug 2015 14:04:57 +0300 
## redis backup every 15 minutes 
## */15 * * * * redis.backup.maintenance.sh >> /var/log/redis.backup.log 2>&1 
## at /etc/rc.local : 
## test -d /var/run/redis.backup.lock.dir && rm -rf /var/run/redis.backup.lock.dir 
## watch the job: 
## tail -f /var/log/redis.backup.log 

#redis-cli LASTSAVE | awk '{print $1}' | { read gmt ; date "+%Y-%m-%d %H:%M:%S" -d "@$gmt" ; } 
# 2015-08-07 01:25:54 

lockf="/var/run/redis.backup.lock.dir"

# check for running script start 
if [ -d "${lockf}" ]
then
 echo "$(date +%Y-%m-%d.%H.%M.%S) : ${lockf} exists, exiting"
 exit 0
else
 mkdir "${lockf}" && echo "$(date +%Y-%m-%d.%H.%M.%S) : created lock at ${lockf}"
fi


echo "$(date +%Y-%m-%d.%H.%M.%S) : redis backup start"
echo "$(date +%Y-%m-%d.%H.%M.%S) : cleanup the /redis_backups and leave the last 6 backups"
find /redis_backups -maxdepth 1 -type f -name "dump.rdb.*" | sort -r | sed '7,$!d' | while read to_be_deleted; do rm -f ${to_be_deleted} && echo "$(date +%Y-%m-%d.%H.%M.%S) : deleted ${to_be_deleted}";done

last_save=$(redis-cli LASTSAVE | awk '{print $1}')
echo -n "$(date +%Y-%m-%d.%H.%M.%S) : executing redis-cli BGSAVE : "
redis-cli BGSAVE
while true
do
 if [ $(redis-cli LASTSAVE | awk '{print $1}') -eq ${last_save} ]
 then
 echo -n ". "
 sleep 2
 else
 echo ""
 echo "$(date +%Y-%m-%d.%H.%M.%S) : start ionice -c2 -n0 cp -vv /opt/redis/dump.rdb to /redis_backups/"
 ionice -c2 -n0 cp -vv /opt/redis/dump.rdb /redis_backups/dump.rdb.$(date +%Y-%m-%d.%H.%M.%S) && echo "$(date +%Y-%m-%d.%H.%M.%S) : backup comleted"
 break
 fi
done


if [ -d "${lockf}" ]
then
 echo "$(date +%Y-%m-%d.%H.%M.%S) : removing the lock"
 rm -rf "${lockf}"
fi
Advertisements
Categories: AWS, bash, NoSQL Tags: , ,

Drupal install of commerce_kickstart-7.x-1.8-core at AWS on two PostgreSQL 9.1.4 replicated nodes with Nginx and php-fpm

July 22, 2012 Leave a comment

I decided to try Drupal on two PostgreSQL nodes replicated as master and readonly slave.
The usual choice for webserver is nginx

Follow the http://drupal.org/documentation/install and
http://drupal.org/project/commerce_kickstart
http://wiki.nginx.org/Drupal

# get the source and install it at the proper location like /usr/share/nginx/drupal
wget  http://ftp.drupal.org/files/projects/disable_messages-7.x-1.0.tar.gz

#set the proper nginx config 
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 15:36:27]:[/var/lib/pgsql]$ cat /etc/nginx/conf.d/drupal.conf
server {
        server_name darkstar.gotdns.org;
#        root /var/lib/pgsql9/xfs.data/drupal; 
        root /usr/share/nginx/drupal; ## <-- Your only path reference.
 
        location = /favicon.ico {
                log_not_found off;
                access_log off;
        }
 
        location = /robots.txt {
                allow all;
                log_not_found off;
                access_log off;
        }
 
        # This matters if you use drush
        location = /backup {
                deny all;
        }
 
        # Very rarely should these ever be accessed outside of your lan
        location ~* \.(txt|log)$ {
                allow 10.210.213.55/32;
                deny all;
        }
 
        location ~ \..*/.*\.php$ {
                return 403;
        }
 
        location / {
                # This is cool because no php is touched for static content
                try_files $uri @rewrite;
        }
 
        location @rewrite {
                # Some modules enforce no slash (/) at the end of the URL
                # Else this rewrite block wouldn't be needed (GlobalRedirect)
                rewrite ^/(.*)$ /index.php?q=$1;
        }
 
        location ~ \.php$ {
	        fastcgi_pass   127.0.0.1:9000;
                fastcgi_split_path_info ^(.+\.php)(/.+)$;
                #NOTE: You should have "cgi.fix_pathinfo = 0;" in php.ini
                include fastcgi_params;
                fastcgi_param SCRIPT_FILENAME $document_root$fastcgi_script_name;
                fastcgi_intercept_errors on;
		#fastcgi_pass   unix:/var/run/php-fpm.sock;
        	fastcgi_index  index.php;
		fastcgi_param  SCRIPT_NAME        $fastcgi_script_name;

        }
 
        # Fighting with ImageCache? This little gem is amazing.
        location ~ ^/sites/.*/files/imagecache/ {
                try_files $uri @rewrite;
        }
        # Catch image styles for D7 too.
        location ~ ^/sites/.*/files/styles/ {
                try_files $uri @rewrite;
        }
 
        location ~* \.(js|css|png|jpg|jpeg|gif|ico)$ {
                expires max;
                log_not_found off;
        }
}

#check the nginx config :
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 15:42:02]:[/var/lib/pgsql]$ nginx -t -c /etc/nginx/nginx.conf
nginx: the configuration file /etc/nginx/nginx.conf syntax is ok
nginx: configuration file /etc/nginx/nginx.conf test is successful

# and reload it
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 15:44:05]:[/var/lib/pgsql]$ service  nginx reload
Reloading nginx:  

# create pgsql user and db and set the pemirssions
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 11:55:33]:[~]$ createuser -U postgres -h 127.0.0.1 --pwprompt --encrypted --no-createrole --no-createdb drupal 
Enter password for new role: 
Enter it again: 
Shall the new role be a superuser? (y/n) n
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 12:06:25]:[~]$ createdb -U postgres -h 127.0.0.1 --encoding=UTF8 --owner=drupal drupal

# set the psql user permissions at the slave with same password you set to the drupal user at the master:
root@ip-10-28-82-97-node-2:[Sun Jul 22 16:01:59]:[/var/lib/pgsql/data]$ cat  /var/lib/pgsql/data/pg_hba.conf | grep -i drupal
hostnossl       drupal          drupal             masterIP/32	 password

# ativate the user permissions 
root@ip-10-28-82-97-node-2:[Sun Jul 22 14:23:31]:[/var/lib/pgsql/data]$ service postgresql reload

#check the connection to slave:
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 16:03:33]:[/usr/share/nginx]$ echo "select version();" | psql -h slaveIP -U drupal drupal
Password for user drupal: 
                                                   version                                                   
-------------------------------------------------------------------------------------------------------------
 PostgreSQL 9.1.4 on x86_64-redhat-linux-gnu, compiled by gcc (GCC) 4.4.6 20110731 (Red Hat 4.4.6-3), 64-bit
(1 row)


#start the install process hitting the index.php
#in case you get <strong>Error not found</strong>  change the permissions of the drupal root directory to 0755:   
chmod -R 0755  /usr/share/nginx/drupal

# install some of the php modules requested by the install as gd, dom and mbstring:
yum install php-soap php-mbstring php-xml  -y
#then restart the php-fpm 
root@domU-12-31-39-09-D2-CD-node-1:[Sun Jul 22 15:09:19]:[/var/lib/pgsql]$ service  php-fpm restart
Stopping php-fpm:                                          [  OK  ]
Starting php-fpm:                                          [  OK  ]

if you get an Warning message like this one:
Compilation failed: unknown option bit(s) set at offset 0 in user_validate_name() (line 645 of /usr/share/nginx/drupal/modules/user/user.module).

this is due to php PECL pcre extension update
there will be some more similar errors indicated at
+1606 /usr/share/nginx/drupal/includes/bootstrap.inc
+3726 /usr/share/nginx/drupal/includes/common.inc
+645 /usr/share/nginx/drupal/modules/user/user.module

the fastest way to fix it at AWS is to do yum update of pcre.x86_64

in general with AWS is better to do just yum update

then, everything works as expected,

do not forget to set the permissions of ./sites/default/settings.php to 0755 after the install

how to set the drupal to use the slave :
thank to padraig o’sullivan who posted
The settings.php file for the Drupal site needs to be updated to know about this slave database.

end of the story, now we have possible web cluster as the second node could be used for second web server, just some db connection settings have to be adjusted. and NFS with CacheFS for the web writable part of the drupal.

The drupal admin is owned by my daughter now … she is about to create the online store.

Post install check of plproxy-2.4 on replicated PostgreSQL 9.1.3 at Amazon Web Services (EC2 , S3)

May 18, 2012 Leave a comment

I got an error on the post install check of plproxy:


root@domU-12-31-39-07-39-AE-node-1:[Fri May 18 18:37:24]:[/opt/installs/plproxy-2.4]$ make installcheck postgres
/usr/lib64/pgsql/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=. --psqldir='/usr/bin'   --dbname=regression --inputdir=test --dbname=regression plproxy_init plproxy_test plproxy_select plproxy_many plproxy_errors plproxy_clustermap plproxy_dynamic_record plproxy_encoding plproxy_split plproxy_target plproxy_sqlmed
(using postmaster on Unix socket, default port)
============== dropping database "regression"         ==============
DROP DATABASE
============== creating database "regression"         ==============
CREATE DATABASE
ALTER DATABASE
============== running regression test queries        ==============
test plproxy_init             ... ok
test plproxy_test             ... FAILED
test plproxy_select           ... ok
test plproxy_many             ... ok
test plproxy_errors           ... ok
test plproxy_clustermap       ... ok
test plproxy_dynamic_record   ... ok
test plproxy_encoding         ... FAILED (test process exited with exit code 2)
test plproxy_split            ... ok
test plproxy_target           ... ok
test plproxy_sqlmed           ... ok

=======================
 2 of 11 tests failed. 
=======================

The differences that caused some tests to fail can be viewed in the
file "/opt/installs/plproxy-2.4/regression.diffs".  A copy of the test summary that you see
above is saved in the file "/opt/installs/plproxy-2.4/regression.out".

make: *** [installcheck] Error 1

how to fix plproxy_test:
Edit the plproxy Make file and add to REGRESS_OPTS proper values in a view to use the correct user and host


root@domU-12-31-39-07-39-AE-node-1:[Fri May 18 19:10:12]:[/opt/installs/plproxy-2.4]$ fgrep REGRESS_OPTS  Makefile
REGRESS_OPTS = --dbname=regression --inputdir=test --user=postgres --host=127.0.0.1
# now we have :

root@domU-12-31-39-07-39-AE-node-1:[Fri May 18 19:05:51]:[/opt/installs/plproxy-2.4]$ make installcheck 
/usr/lib64/pgsql/pgxs/src/makefiles/../../src/test/regress/pg_regress --inputdir=. --psqldir='/usr/bin'   --dbname=regression --inputdir=test --user=postgres --host=127.0.0.1 --no-locale --dbname=regression plproxy_init plproxy_test plproxy_select plproxy_many plproxy_errors plproxy_clustermap plproxy_dynamic_record plproxy_encoding plproxy_split plproxy_target plproxy_sqlmed
(using postmaster on 127.0.0.1, default port)
============== dropping database "regression"         ==============
DROP DATABASE
============== creating database "regression"         ==============
CREATE DATABASE
ALTER DATABASE
============== running regression test queries        ==============
test plproxy_init             ... ok
test plproxy_test             ... ok
test plproxy_select           ... ok
test plproxy_many             ... ok
test plproxy_errors           ... ok
test plproxy_clustermap       ... ok
test plproxy_dynamic_record   ... ok
test plproxy_encoding         ... FAILED (test process exited with exit code 2)
test plproxy_split            ... ok
test plproxy_target           ... ok
test plproxy_sqlmed           ... ok

=======================
 1 of 11 tests failed. 
=======================


Compile and install PostGIS 2.0.0 on replicated PostgreSQL 9.1.3 at Amazon Web Services (EC2 , S3)

May 18, 2012 6 comments

How to install that :

1. get the source needed


cd /opt/installs
wget http://postgis.refractions.net/download/postgis-2.0.0.tar.gz
wget http://download.osgeo.org/proj/proj-4.8.0.tar.gz
wget http://download.osgeo.org/geos/geos-3.3.3.tar.bz2
git clone https://github.com/json-c/json-c.git
wget http://download.osgeo.org/gdal/gdal-1.9.1RC2.tar.gz
wget http://sourceforge.net/projects/cunit/files/latest/download
wget http://prdownloads.sourceforge.net/dblatex/dblatex-0.3.2.tar.bz2?download

# NOTE, you may skip the CUnit and dblatext install, but you will get these warnings at the configure of the postgs:
# ==========================================================================================
# checking for dblatex... no
# configure: WARNING: dblatex is not installed so PDF documentation cannot be built
# checking CUnit/CUnit.h usability... no
# checking CUnit/CUnit.h presence... no
# checking for CUnit/CUnit.h... no
# configure: WARNING: could not locate CUnit required for liblwgeom unit tests
# ==========================================================================================

yum install tetex-tex4ht -y
yum install libxml2-devel -y
yum install ImageMagick* -y

2. Compile and install proj-4


tar xvfz proj-4.8.0.tar.gz
cd proj-4.8.0
./configure ; make -j3 install; ldconfig
# if you run 64bit distro, the postgis build will fail so link the proj.pc at the 64bit pkgconfig directory :  
ln -s /usr/local/lib/pkgconfig/proj.pc /usr/lib64/pkgconfig/


3. Compile and install geos:
in case you dont have installed c++, install it


yum install gcc-c++ -y
tar xvf geos-3.3.3.tar.bz2 
cd geos-3.3.3
./configure --disable-libtool-lock
make -j3  install; ldconfig

4. Compile and install json-c lib:


cd /opt/install
cd cd json-c/
./autogen.sh
./configure
make -j3 install; ldconfig
# if you run 64bit distro, link the jspn.pc at the 64bit pkgconfig directory :  
ln -s /usr/local/lib/pkgconfig/json.pc /usr/lib64/pkgconfig/

4. Compile and install gdal-1.9.1 lib:
gdal compilation takes some time, so its better to detach the make process:


cd /opt/installs
tar xvf gdal-1.9.1RC2.tar.gz
cd gdal-1.9.1
./autogen.sh 
./configure
echo "make -j3 install; ldconfig | tee /tmp/gdal-1.9.1RC2.install.log" | at now 
tail -f /tmp/gdal-1.9.1RC2.install.log

6. Compile and install CUnit-2.1-2:

cd /opt/installs
tar xf CUnit-2.1-2-src.tar.bz2
cd CUnit-2.1-2
./configure  ; make -j3 install; ldconfig
# if you run 64bit distro, link the cunit.pc at the 64bit pkgconfig directory :  
ln -s /usr/local/lib/pkgconfig/cunit.pc /usr/lib64/pkgconfig/

6. Compile and install postgis-2.0.0


cd /opt/installs
tar xvfz postgis-2.0.0.tar.gz
cd postgis-2.0.0
./configure
# at the end you will see something like:
#
#  PostGIS is now configured for x86_64-unknown-linux-gnu
#
# -------------- Compiler Info ------------- 
#  C compiler:           gcc -g -O2
#  C++ compiler:         g++ -g -O2
#
# -------------- Dependencies -------------- 
#  GEOS config:          /usr/local/bin/geos-config
#  GEOS version:         3.3.3
#  GDAL config:          /usr/local/bin/gdal-config
#  GDAL version:         1.9.1
#  PostgreSQL config:    /usr/bin/pg_config
#  PostgreSQL version:   PostgreSQL 9.1.3
#  PROJ4 version:        48
#  Libxml2 config:       /usr/bin/xml2-config
#  Libxml2 version:      2.7.6
#  JSON-C support:       yes
#  PostGIS debug level:  0
#  Perl:                 /usr/bin/perl
#
# --------------- Extensions --------------- 
#  PostGIS Raster:       enabled
#  PostGIS Topology:     enabled
#
# -------- Documentation Generation -------- 
#  xsltproc:             /usr/bin/xsltproc
#  xsl style sheets:     /usr/share/sgml/docbook/xsl-stylesheets
#  dblatex:              /usr/bin/dblatex
#  convert:              /usr/bin/convert
#  mathml2.dtd:          http://www.w3.org/Math/DTD/mathml2/mathml2.dtd
#
make -j3

There is huge possibility the build process to fail with the following error:


gcc -O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector --param=ssp-buffer-size=4 -m64 -mtune=generic -DLINUX_OOM_ADJ=0 -Wall -Wmissing-prototypes -Wpointer-arith -Wdeclaration-after-statement -Wendif-labels -Wformat-security -fno-strict-aliasing -fwrapv -fpic -I/usr/local/include   -I/usr/include/libxml2 -I../liblwgeom -I../libpgcommon -I. -I. -I/usr/include/pgsql/server -I/usr/include/pgsql/internal -D_GNU_SOURCE -I/usr/include/libxml2   -c -o lwgeom_in_geojson.o lwgeom_in_geojson.c
In file included from lwgeom_in_geojson.c:31:
/usr/local/include/json/json.h:27:34: error: json_object_iterator.h: No such file or directory
lwgeom_in_geojson.c:467: warning: no previous prototype for ‘postgis_libjson_version’
make[1]: *** [lwgeom_in_geojson.o] Error 1
make[1]: Leaving directory `/opt/installs/postgis-2.0.0/postgis'
make: *** [all] Error 1
root@ip-10-28-187-67-node-2:[Fri May 18 13:01:52]:[/opt/installs/postgis-2.0.0]$

The issue is the following :

line 27 of /usr/local/include/json/json.h defined json_object_iterator.h to be included, but that file is missing in the include directory:


root@ip-10-28-187-67-node-2:[Fri May 18 15:57:29]:[/var/log]$ sed '27,27!d' /usr/local/include/json/json.h 
#include "json_object_iterator.h"
root@ip-10-28-187-67-node-2:[Fri May 18 13:03:14]:[/opt/installs/postgis-2.0.0]$ ls -lrht /usr/local/include/json/
total 72K
-rw-r--r-- 1 root root 6.0K May 18 12:54 json_tokener.h
-rw-r--r-- 1 root root 1.1K May 18 12:54 json_object_private.h
-rw-r--r-- 1 root root  17K May 18 12:54 json_object.h
-rw-r--r-- 1 root root  579 May 18 12:54 json_inttypes.h
-rw-r--r-- 1 root root  671 May 18 12:54 json.h
-rw-r--r-- 1 root root  163 May 18 12:54 json_config.h
-rw-r--r-- 1 root root 1.7K May 18 12:54 debug.h
-rw-r--r-- 1 root root  693 May 18 12:54 bits.h
-rw-r--r-- 1 root root 1.1K May 18 12:54 arraylist.h
-rw-r--r-- 1 root root 2.2K May 18 12:54 printbuf.h
-rw-r--r-- 1 root root 6.6K May 18 12:54 linkhash.h
-rw-r--r-- 1 root root  971 May 18 12:54 json_util.h

root@ip-10-28-187-67-node-2:[Fri May 18 13:03:11]:[/opt/installs/postgis-2.0.0]$ locate json_object_iterator.h
/opt/installs/json-c/json_object_iterator.h

#how to fix the issue :
cp -vvv /opt/installs/json-c/json_object_iterator.h /usr/local/include/json/

so, clean the build and do it again:


cd /opt/installs/postgis-2.0.0
make clean 
./configure
echo "make -j3 install | tee -a /tmp/postgis-2.0.0.install.log" | at now 
tail -f /tmp/postgis-2.0.0.install.log

that is all for the install

now, we have to test it , before to claim it will work:
NOTE: do not test on a slave node in case the replication is active, the test will fail :

make test 
.....
Creating spatial db postgis_reg 
createdb: database creation failed: ERROR:  cannot execute CREATE DATABASE in a read-only transaction
createlang: could not connect to database postgis_reg: FATAL:  database "postgis_reg" does not exist
psql: FATAL:  database "postgis_reg" does not exist
dropdb: database removal failed: ERROR:  cannot execute DROP DATABASE in a read-only transaction
Preparing spatial db postgis_reg 

 Something went wrong during db initialization (core module).
 For details, check /tmp/pgis_reg/regress_log

dropdb: database removal failed: ERROR:  cannot execute DROP DATABASE in a read-only transaction
make[1]: *** [check] Error 1
make[1]: Leaving directory `/opt/installs/postgis-2.0.0/regress'
make: *** [check] Error 1

test on the master:

# Note at the test time the slave will be catching up 
# Start the test detached as it will take some time, at AWS micro instance it takes a lot 🙂  
echo "make -j3 test | tee -a /tmp/postgis-2.0.0.make.test.log" | at now 
tail -f  /tmp/postgis-2.0.0.make.test.log
....
.....
Creating spatial db postgis_reg 
Preparing spatial db postgis_reg 
TMPDIR is /tmp/pgis_reg
PATH is /usr/bin:/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:/opt/aws/bin:/root/bin

 PostgreSQL 9.1.3 on x86_64-redhat-linux-gnu, compiled by gcc (GCC) 4.4.6 20110731 (Red Hat 4.4.6-3), 64-bit
 Postgis 2.0.0 - r9605 - 2012-05-18 17:38:33
   GEOS: 3.3.3-CAPI-1.7.4
   PROJ: Rel. 4.8.0, 6 March 2012

Running tests

 loader/Point ....................... ok 
 loader/PointM ....................... ok 
 loader/PointZ .....
....

postgres@kida:5432::postgres=[Fri May 18 12:32:41 EDT 2012]# \l
                                    List of databases
     Name      |  Owner   | Encoding |   Collate   |    Ctype    |   Access privileges   
---------------+----------+----------+-------------+-------------+-----------------------
 kannel        | mtel     | UTF8     | en_US.UTF-8 | en_US.UTF-8 | 
 postgis_reg   | root     | UTF8     | C           | en_US.UTF-8 | 
 postgres      | postgres | UTF8     | en_US.UTF-8 | en_US.UTF-8 | 
 regression    | root     | UTF8     | en_US.UTF-8 | en_US.UTF-8 | 


postgres@juan:5432::postgres=[Fri May 18 12:37:36 EDT 2012]# select pg_current_xlog_location();
 pg_current_xlog_location 
--------------------------
 0/C7AD518
(1 row)

Time: 500.731 ms

postgres@kida:5432::postgres=[Fri May 18 12:37:38 EDT 2012]# select pg_last_xlog_replay_location()
;
 pg_last_xlog_replay_location 
------------------------------
 0/C7AD518
(1 row)

Time: 162.957 ms

And that is all, you have now a Master – slave cluster with the Latest PostGIS ready for the devs ..:)

cheers

aaand …
Slackware4LIFE 🙂

PostgreSQL 9.1.3 streaming replication at Amazon Web Services (EC2 , S3)

May 17, 2012 Leave a comment

1. You need at least two nodes – Master and 1 slave
The solution is :

– Create ami image from the first instance, create second one and load the fresh new ami image with same security profile
– Check / add the port permissions at the security profile.
– change the PS1 at the two nodes to contain the strings node-1 and node-2 in a view be aware at which node are you in every single moment.

2. Follow the instructions from the official PostgreSQL wiki
– use the Amazon internal IP addresses in the replication setup
– Make sure the replication is working:

# Master
postgres@juan:5432::postgres=[Wed May 16 19:40:30 EDT 2012]# SELECT pg_current_xlog_location();
 pg_current_xlog_location 
--------------------------
 0/9001280
(1 row)

Time: 161.324 ms

#Slave
postgres@kida:5432::postgres=[Wed May 16 19:00:42 EDT 2012]# select pg_last_xlog_replay_location();
 pg_last_xlog_replay_location 
------------------------------
 0/90011F0
(1 row)

Time: 165.566 ms

activating the XFS partition previously created (Add XFS partitiion to be used by PostgreSQL at AWS EC2)

# Master
postgres@juan:5432::postgres=[Wed May 16 19:00:28 EDT 2012]# create tablespace xfs location '/var/lib/pgsql9/xfs.data' ;
CREATE TABLESPACE
postgres@juan:5432::postgres=[Wed May 16 19:00:35 EDT 2012]# \db
               List of tablespaces
    Name    |  Owner   |         Location         
------------+----------+--------------------------
 pg_default | postgres | 
 pg_global  | postgres | 
 xfs        | postgres | /var/lib/pgsql9/xfs.data
(3 rows)

#The new table stace is replicated at the slave as well:
postgres@kida:5432::postgres=[Wed May 16 19:00:13 EDT 2012]# \db
       List of tablespaces
    Name    |  Owner   | Location 
------------+----------+----------
 pg_default | postgres | 
 pg_global  | postgres | 
(2 rows)

postgres@kida:5432::postgres=[Wed May 16 19:00:15 EDT 2012]# \db
               List of tablespaces
    Name    |  Owner   |         Location         
------------+----------+--------------------------
 pg_default | postgres | 
 pg_global  | postgres | 
 xfs        | postgres | /var/lib/pgsql9/xfs.data
(3 rows)

Add XFS partitiion to be used by PostgreSQL at AWS EC2

May 11, 2012 2 comments

1. Use the AWS Elastick Block Store to create new volume in the same zone as the instance where your PostgreSQL is.
2. Attach the new volume to the instance
When its done, you will see in the dmesg output something like :

[ 5192.466502] blkfront: xvdf: barrier or flush: disabled
[ 5192.469114]  xvdf: unknown partition table

3. Install xfs utilities

root@domU-12-31-39-07-39-AE:[Fri May 11 11:15:37]:[/var/log]$ yum search xfs
Loaded plugins: fastestmirror, priorities, security, update-motd
Loading mirror speeds from cached hostfile
 * amzn-main: packages.us-east-1.amazonaws.com
 * amzn-updates: packages.us-east-1.amazonaws.com
=============================================================================== N/S Matched: xfs ================================================================================
xfsprogs.i686 : Utilities for managing the XFS filesystem
xfsprogs.x86_64 : Utilities for managing the XFS filesystem
xfsprogs-devel.x86_64 : XFS filesystem-specific static libraries and headers
xfsprogs-qa-devel.x86_64 : XFS QA filesystem-specific static libraries and headers
xorg-x11-xfs.x86_64 : X.Org X11 xfs font server
xorg-x11-xfs-utils.x86_64 : X.Org X11 font server utilities

  Name and summary matches only, use "search all" for everything.
root@domU-12-31-39-07-39-AE:[Fri May 11 11:16:17]:[/var/log]$ yum install xfsprogs* -y 
Loaded plugins: fastestmirror, priorities, security, update-motd
Loading mirror speeds from cached hostfile
 * amzn-main: packages.us-east-1.amazonaws.com
 * amzn-updates: packages.us-east-1.amazonaws.com
Setting up Install Process
Resolving Dependencies
--> Running transaction check
---> Package xfsprogs.x86_64 0:3.1.3-1.7.amzn1 will be installed
---> Package xfsprogs-devel.x86_64 0:3.1.3-1.7.amzn1 will be installed
---> Package xfsprogs-qa-devel.x86_64 0:3.1.3-1.7.amzn1 will be installed
--> Finished Dependency Resolution

Dependencies Resolved

=================================================================================================================================================================================
 Package                                        Arch                                Version                                         Repository                              Size
=================================================================================================================================================================================
Installing:
 xfsprogs                                       x86_64                              3.1.3-1.7.amzn1                                 amzn-main                              1.4 M
 xfsprogs-devel                                 x86_64                              3.1.3-1.7.amzn1                                 amzn-main                               29 k
 xfsprogs-qa-devel                              x86_64                              3.1.3-1.7.amzn1                                 amzn-main                               92 k

Transaction Summary
=================================================================================================================================================================================
Install       3 Package(s)

Total download size: 1.5 M
Installed size: 3.6 M
Downloading Packages:
(1/3): xfsprogs-3.1.3-1.7.amzn1.x86_64.rpm                                                                                                                | 1.4 MB     00:00     
(2/3): xfsprogs-devel-3.1.3-1.7.amzn1.x86_64.rpm                                                                                                          |  29 kB     00:00     
(3/3): xfsprogs-qa-devel-3.1.3-1.7.amzn1.x86_64.rpm                                                                                                       |  92 kB     00:00     
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Total                                                                                                                                            4.4 MB/s | 1.5 MB     00:00     
Running rpm_check_debug
Running Transaction Test
Transaction Test Succeeded
Running Transaction
  Installing : xfsprogs-3.1.3-1.7.amzn1.x86_64                                                                                                                               1/3 
  Installing : xfsprogs-devel-3.1.3-1.7.amzn1.x86_64                                                                                                                         2/3 
  Installing : xfsprogs-qa-devel-3.1.3-1.7.amzn1.x86_64                                                                                                                      3/3 

Installed:
  xfsprogs.x86_64 0:3.1.3-1.7.amzn1                    xfsprogs-devel.x86_64 0:3.1.3-1.7.amzn1                    xfsprogs-qa-devel.x86_64 0:3.1.3-1.7.amzn1                   

Complete!
root@domU-12-31-39-07-39-AE:[Fri May 11 11:16:59]:[/var/log]

4. Create the mount point


mkdir /var/lib/pgsql9/xfs.data

5. Format the new volume


mkfs.xfs -f /dev/xvdf

root@domU-12-31-39-07-39-AE:[Fri May 11 11:29:54]:[/var/log]$ mkfs.xfs -f /dev/xvdf
meta-data=/dev/xvdf              isize=256    agcount=4, agsize=1966080 blks
         =                       sectsz=512   attr=2
data     =                       bsize=4096   blocks=7864320, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0
log      =internal log           bsize=4096   blocks=3840, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0

6. Mount the formatted volume


mount -t xfs -o inode64,nobarrier,noatime /dev/xvdf  /var/lib/pgsql9/xfs.data

7. Make sure the volume will be mounted at boot time :
– get the mount properties


root@domU-12-31-39-07-39-AE:[Fri May 11 11:30:20]:[/var/log]$ cat /proc/mounts | grep /dev/xvdf
/dev/xvdf /var/lib/pgsql9/xfs.data xfs rw,noatime,attr2,delaylog,nobarrier,inode64,noquota 0 0

– add that line to the fstab


echo "/dev/xvdf /var/lib/pgsql9/xfs.data xfs rw,noatime,attr2,delaylog,nobarrier,inode64,noquota 0 0" >> /etc/fstab