C# Hadoop學習筆記(一)—環境安裝
阿新 • • 發佈:2019-01-03
一、安裝環境
1,前期準備:官網下載“NuGet Package Manager”,按自己已有的VS環境下載對應版本;
3,安裝。
4,通過HDInsight,安裝Windows Azure,目前是預覽版本。
二、測試DEMO
using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using Microsoft.Hadoop; using Microsoft.Hadoop.MapReduce; using Microsoft.Hadoop.WebClient.WebHCatClient; using System.Diagnostics; using System.IO; using System.IO.MemoryMappedFiles; namespace HadoopConsol { class Program { static void Main(string[] args) { Stopwatch sw = new Stopwatch(); long hadoopTime=0; long normalTime=0; sw.Start(); //start hadoop Console.WriteLine(" Hadoop Process Strating ...."); #region Hadoop time #region hadoopconnet Console.WriteLine(" Hadoop Connect Strating ...."); //establish job configuration HadoopJobConfiguration myConfig = new HadoopJobConfiguration(); myConfig.InputPath = "/demo/simple/in"; myConfig.OutputFolder = "/demo/simple/out"; //connect to cluster Uri myUri = new Uri("http://localhost"); string userName = "hadoop"; string passWord = null; IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord); hadoopTime += sw.ElapsedMilliseconds; Console.WriteLine(" Hadoop Connect End."); Console.WriteLine(" Hadoop Connect time:" + sw.ElapsedMilliseconds); #endregion #region hadoopmapreduce sw.Reset(); sw.Start(); Console.WriteLine(" Hadoop MapReduce Strating ...."); //execute mapreduce job MapReduceResult jobResult = myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig); hadoopTime += sw.ElapsedMilliseconds; Console.WriteLine(" Hadoop MapReduce End."); Console.WriteLine(" Hadoop MapReduce Time:"+sw.ElapsedMilliseconds); #endregion #region Hadoop End sw.Reset(); sw.Start(); Console.WriteLine(" Hadoop Endprocess Strating ...."); //write job result to console int exitCode = jobResult.Info.ExitCode; string exitStatus = "Failure"; if (exitCode == 0) exitStatus = "Success"; exitStatus = exitCode + " (" + exitStatus + ")"; Console.WriteLine(); Console.Write("Exit Code = " + exitStatus); Console.WriteLine(" Hadoop Endprocess End."); hadoopTime += sw.ElapsedMilliseconds; Console.WriteLine(" Hadoop Exit Time:" + sw.ElapsedMilliseconds); Console.WriteLine(" Hadoop Process All Time:" + hadoopTime); #endregion #endregion #region Normal time //start Normal Console.WriteLine(" Normal Process Strating ...."); sw.Reset(); sw.Start(); //normal process #region Normal Process int myevenCount = 0; int myeventSum = 0; int myoddCount = 0; int myoddSum = 0; StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt"); while (fs.Peek() >= 0) { string strTemp = fs.ReadLine(); if (Int32.Parse(strTemp) % 2 == 0) { myevenCount++; myeventSum += Int32.Parse(strTemp); } else { myoddCount++; myoddSum += Int32.Parse(strTemp); } } //MemoryMappedFile m = MemoryMappedFile. Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum); Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum); #endregion Console.WriteLine(" Normal Process End."); normalTime += sw.ElapsedMilliseconds; Console.WriteLine(" Normal Exit Time:" + sw.ElapsedMilliseconds); Console.WriteLine(" Normal Process All Time:" + normalTime); #endregion sw.Stop(); Console.Read(); } } public class MySimpleMapper : MapperBase { public override void Map(string inputLine, MapperContext context) { //interpret the incoming line as an integer value int value = int.Parse(inputLine); //determine whether value is even or odd string key = (value % 2 == 0) ? "even" : "odd"; //output key assignment with value context.EmitKeyValue(key, value.ToString()); } } public class MySimpleReducer : ReducerCombinerBase { public override void Reduce( string key, IEnumerable<string> values, ReducerCombinerContext context ) { //initialize counters int myCount = 0; int mySum = 0; //count and sum incoming values foreach (string value in values) { mySum += int.Parse(value); myCount++; } //output results context.EmitKeyValue(key, myCount + "\t" + mySum); } } }