groovy 2.1 and maven

This is pom file example how to add groovy support to java project.

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.middlesphere</groupId>
    <artifactId>hello01</artifactId>
    <packaging>jar</packaging>
    <version>0.1</version>
    <name>hello01</name>
    <url>http://maven.apache.org</url>

    <properties>
        <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
        <gmavenVersion>1.4</gmavenVersion>
        <gmavenProviderSelection>2.0</gmavenProviderSelection>
        <groovyVersion>2.1.6</groovyVersion>
    </properties>

    <dependencies>
        <dependency>
            <groupId>org.codehaus.groovy</groupId>
            <artifactId>groovy-all</artifactId>
            <version>2.1.6</version>
        </dependency>

        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.8.2</version>
            <scope>test</scope>
        </dependency>
    </dependencies>
    <build>
        <plugins>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-compiler-plugin</artifactId>
                <configuration>
                    <target>1.6</target>
                    <source>1.6</source>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.codehaus.gmaven</groupId>
                <artifactId>gmaven-plugin</artifactId>
                <version>${gmavenVersion}</version>
                <configuration>
                    <providerSelection>${gmavenProviderSelection}</providerSelection>
                    <sourceEncoding>UTF-8</sourceEncoding>
                    <source/>
                </configuration>
                <executions>
                    <execution>
                        <goals>
                            <goal>generateStubs</goal>
                            <goal>compile</goal>
                            <goal>generateTestStubs</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
                <dependencies>
                    <dependency>
                        <groupId>org.codehaus.groovy</groupId>
                        <artifactId>groovy-all</artifactId>
                        <version>${groovyVersion}</version>
                    </dependency>
                </dependencies>
            </plugin>

            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-dependency-plugin</artifactId>
                <executions>
                    <execution>
                        <id>copy-dependencies</id>
                        <phase>prepare-package</phase>
                        <goals>
                            <goal>copy-dependencies</goal>
                        </goals>
                        <configuration>
                            <outputDirectory>${project.build.directory}/lib</outputDirectory>
                            <overWriteReleases>false</overWriteReleases>
                            <overWriteSnapshots>false</overWriteSnapshots>
                            <overWriteIfNewer>true</overWriteIfNewer>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-jar-plugin</artifactId>
                <configuration>
                    <archive>
                        <manifest>
                            <addClasspath>true</addClasspath>
                            <classpathPrefix>lib/</classpathPrefix>
                            <mainClass>com.middlesphere.HelloApp</mainClass>
                        </manifest>
                    </archive>
                </configuration>
            </plugin>
        </plugins>
    </build>
</project>

hadoop: set file permissions to a user

#Before user can write to hdfs we need to set permissions

sudo -u hdfs hadoop fs -mkdir /user/username
sudo -u hdfs hadoop fs -chown username:usergroup /user/
username

#if we need to write to hdfs from remote machine ( append avro file, for example )

       //if user exist it return its ugi
       UserGroupInformation ugi = UserGroupInformation.createRemoteUser("username");
        ugi.doAs(new PrivilegedExceptionAction() {
            public Void run() throws Exception {

                Configuration conf = new Configuration();
                conf.set("fs.defaultFS", "hdfs://hadoop-server");

                FileSystem fs = FileSystem.get(conf);

                DatumWriter xmlOrderInfoWriter = new SpecificDatumWriter(XmlOrderInfo.class);
                DataFileWriter dataFileWriter = new DataFileWriter(xmlOrderInfoWriter);

                Path filePath = new Path("/user//data/myfolder/test-xml-files.avro");
                OutputStream out = fs.append(filePath);

                dataFileWriter.appendTo(new FsInput(filePath,conf),out);
                dataFileWriter.append(xmlOrderInfo);
                dataFileWriter.close();
                out.close();
                System.out.println("Test avro file is appended in HDFS successfully");

                return null;
            }

        });

create maven project

Maven

#create maven project
mvn archetype:generate -DgroupId=my.company.project -DartifactId=modulename -DarchetypeArtifactId=maven-archetype-quickstart -Dversion=0.1 -DinteractiveMode=false


GIT

git init
git add *
git commit -a -m"Initial commit"
git remote add origin https://username@bitbucket.org/username/myrepo.git
git push -u origin --all
mvn idea:idea

Expand LVM volume after install

Sometimes it is necessary to expand / partition after default installation and truncate /home partition using LVM.

umount /home
lvremove /dev/VolGroup/lv_home
lvextend -l +100%FREE  /dev/VolGroup/lv_root
resize2fs  /dev/VolGroup/lv_root

delete line about /home in /etc/fstab

Install GNOME after RHEL 6 minimal installation

#if u use oracle repo then
rpm -e redhat-release-server.x86_64 --nodeps
yum install oraclelinux-release.x86_64

#
yum -y groupinstall "X Window System" "Fonts" "Desktop"
vi /etc/inittab
        change id:3:initdefault: to id:5:initdefault:
init 6

RHEL 6 / Centos 6 and Python 3

Here are some instructions, how to install Python 3 on RHEL 6 / Centos 6 .

------------------------------------------------------------------------------------

yum update
yum groupinstall "Development tools"
yum install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel
cd /opt
wget http://python.org/ftp/python/3.3.1/Python-3.3.1.tar.bz2
tar xf Python-3.3.1.tar.bz2
rm Python-3.3.1.tar.bz2
cd Python-3.3.1/
./configure --prefix=/usr/local
make && make altinstall
cd ..
rm -rf ./Python-3.3.1
wget http://pypi.python.org/packages/source/d/distribute/distribute-0.6.36.tar.gz
tar xvf distribute-0.6.36.tar.gz
cd distribute-0.6.36
python3.3 setup.py install
easy_install-3.3 virtualenv
easy_install-3.3 pip
cd ..
rm -rf ./distribute-0.6.36*

Warning - "altinstall" is very important

Safely usage different versions of python on the same machine:

virtualenv-3.3 --distribute projectfolder

source projectfolder/bin/activate

deactivate