@inproceedings{eba2b4120b554d2e86edd6bc27abff6f,
title = "Building a research data science platform from industrial machines",
abstract = "Data Science research has a long history in academia which spans from large-scale data management, to data mining and data analysis using technologies from database management systems (DBMS's). While traditional HPC offers tools on leveraging existing technologies with data processing needs, the large volume of data and the speed of data generation pose significant challenges. Using the Hadoop platform and tools built on top of it drew immense interest from academia after it gained success in industry. Georgia Institute of Technology received a donation of 200 compute nodes from Yahoo. Turning these industrial machines into a research Data Science Platform (DSP) poses unique challenges, such as: nontrivial hardware design decisions, configuration tool choices, node integration into existing HPC infrastructure, partitioning resource to meet different application needs, software stack choices, etc. We have 40 nodes up and running, 24 running as a Hadoop and Spark cluster, 12 running as a HBase and OpenTSDB cluster, the others running as service nodes. We successfully tested it against Spark Machine Learning algorithms using a 88GB image dataset, Spark DataFrame and GraphFrame with a Wikipedia dataset, and Hadoop MapReduce wordcount on a 300GB dataset. The OpenTSDB cluster is for real-time time series data ingestion and storage for sensor data. We are working on bringing up more nodes. We share our first-hand experience gained in our journey, which we believe will benefit and inspire other academic institutions.",
keywords = "Data Science Platform, Hadoop, Machine Learning, Software and Hardware Configuration, Spark",
author = "Liu, \{Fang Cherry\} and Fu Shen and Chau, \{Duen Horng\} and Neil Bright and Mehmet Belgin",
note = "Publisher Copyright: {\textcopyright} 2016 IEEE.; 4th IEEE International Conference on Big Data, Big Data 2016 ; Conference date: 05-12-2016 Through 08-12-2016",
year = "2016",
doi = "10.1109/BigData.2016.7840859",
language = "English",
series = "Proceedings - 2016 IEEE International Conference on Big Data, Big Data 2016",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
pages = "2270--2275",
editor = "James Joshi and George Karypis and Ling Liu and Hu, \{Xiaohua Tony\} and Ronay Ak and Yinglong Xia and Weijia Xu and Aki-Hiro Sato and Sudarsan Rachuri and Lyle Ungar and Yu, \{Philip S.\} and Rama Govindaraju and Toyotaro Suzumura",
booktitle = "Proceedings - 2016 IEEE International Conference on Big Data, Big Data 2016",
address = "United States",
}