您现在的位置 >> Hadoop教程 >> Hadoop实战 >> 专题  
 

C# Hadoop学习笔记(一)

【作者:Hadoop实战专家】【关键词:下载 安装 】 【点击:10665次】【2014-02-0】
1,前期准备:官网下载“NuGet Package Manager”,按自己已有的VS环境下载对应版本;9. string exitStatus = "Failure"; Console.WriteLine(" Hadoop Process All Time:" + hadoopTime); string key = (value % 2 == 0) ?  

相关热门搜索:

大数据标签:hadoop mapreduce hive bigdata

一、安装环境

1,前期准备:官网下载“NuGet Package Manager”,按自己已有的VS环境下载对应版本;

2,利用NuGet下载Hadoop For .NET SDK,地址“http://hadoopsdk.codeplex.com/”

3,安装。

4,通过HDInsight,安装Windows Azure,目前是预览版本。

5,参照网址“http://blogs.msdn.com/b/data_otaku/archive/2013/08/14/hadoop-for-net-developers.aspx” 系统学习API

二、测试DEMO

[csharp] view plaincopy

1. using System;
2. using System.Collections.Generic;
3. using System.Linq;
4. using System.Text;
5. using System.Threading.Tasks;
6. using Microsoft.Hadoop;
7. using Microsoft.Hadoop.MapReduce;
8. using Microsoft.Hadoop.WebClient.WebHCatClient;
9. using System.Diagnostics;
10. using System.IO;
11. using System.IO.MemoryMappedFiles;
12.
13.
14. namespace HadoopConsol
15. {
16.     class Program
17.     {
18.         static void Main(string[] args)
19.         {
20.             Stopwatch sw = new Stopwatch();
21.             long hadoopTime=0;
22.             long normalTime=0;
23.
24.             sw.Start();
25.
26.             //start hadoop
27.             Console.WriteLine("     Hadoop Process Strating ....");
28.
29.             #region Hadoop time
30.
31.             #region hadoopconnet
32.
33.             Console.WriteLine("     Hadoop Connect Strating ....");
34.
35.             //establish job configuration
36.
37.             HadoopJobConfiguration myConfig = new HadoopJobConfiguration();
38.
39.             myConfig.InputPath = "/demo/simple/in";
40.
41.             myConfig.OutputFolder = "/demo/simple/out";
42.
43.
44.
45.             //connect to cluster
46.
47.             Uri myUri = new Uri("http://localhost");
48.
49.             string userName = "hadoop";
50.
51.             string passWord = null;
52.
53.             IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);
54.
55.             hadoopTime += sw.ElapsedMilliseconds;
56.
57.             Console.WriteLine("     Hadoop Connect End.");
58.
59.             Console.WriteLine("     Hadoop Connect time:" + sw.ElapsedMilliseconds);
60.
61.             #endregion
62.
63.
64.             #region hadoopmapreduce
65.
66.             sw.Reset();
67.             sw.Start();
68.
69.             Console.WriteLine("     Hadoop MapReduce Strating ....");
70.
71.             //execute mapreduce job
72.
73.             MapReduceResult jobResult =
74.
75.                 myCluster.MapReduceJob.Execute(myConfig);
76.
77.             hadoopTime += sw.ElapsedMilliseconds;
78.
79.             Console.WriteLine("     Hadoop MapReduce End.");
80.             Console.WriteLine("     Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);
81.
82.             #endregion
83.
84.             #region Hadoop End
85.
86.             sw.Reset();
87.             sw.Start();
88.
89.             Console.WriteLine("     Hadoop Endprocess Strating ....");
90.
91.             //write job result to console
92.
93.             int exitCode = jobResult.Info.ExitCode;
94.
95.
96.
97.             string exitStatus = "Failure";
98.
99.             if (exitCode == 0) exitStatus = "Success";
100.
101.             exitStatus = exitCode + " (" + exitStatus + ")";
102.
103.             Console.WriteLine();
104.
105.             Console.Write("Exit Code = " + exitStatus);
106.
107.             Console.WriteLine("     Hadoop Endprocess End.");
108.             hadoopTime += sw.ElapsedMilliseconds;
109.             Console.WriteLine("     Hadoop Exit Time:" + sw.ElapsedMilliseconds);
110.             Console.WriteLine("     Hadoop Process All Time:" + hadoopTime);
111.             #endregion
112.
113.             #endregion
114.
115.
116.             #region Normal time
117.             //start Normal
118.             Console.WriteLine("     Normal Process Strating ....");
119.
120.             sw.Reset();
121.             sw.Start();
122.
123.             //normal process
124.             #region Normal Process
125.
126.             int myevenCount = 0;
127.             int myeventSum = 0;
128.
129.             int myoddCount = 0;
130.             int myoddSum = 0;
131.
132.             StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");
133.
134.             while (fs.Peek() >= 0)
135.             {
136.                 string strTemp = fs.ReadLine();
137.                 if (Int32.Parse(strTemp) % 2 == 0)
138.                 {
139.                     myevenCount++;
140.                     myeventSum += Int32.Parse(strTemp);
141.                 }
142.                 else
143.                 {
144.                     myoddCount++;
145.                     myoddSum += Int32.Parse(strTemp);
146.                 }
147.             }
148.                //MemoryMappedFile m = MemoryMappedFile.
149.             Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);
150.             Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);
151.
152.             #endregion
153.
154.             Console.WriteLine("     Normal Process End.");
155.
156.             normalTime += sw.ElapsedMilliseconds;
157.             Console.WriteLine("     Normal Exit Time:" + sw.ElapsedMilliseconds);
158.             Console.WriteLine("     Normal Process All Time:" + normalTime);
159.
160.             #endregion
161.
162.             sw.Stop();
163.
164.             Console.Read();
165.
166.         }
167.
168.     }
169.
170.     public class MySimpleMapper : MapperBase
171.     {
172.
173.         public override void Map(string inputLine, MapperContext context)
174.         {
175.
176.             //interpret the incoming line as an integer value
177.
178.             int value = int.Parse(inputLine);
179.
180.             //determine whether value is even or odd
181.
182.             string key = (value % 2 == 0) ? "even" : "odd";
183.
184.             //output key assignment with value
185.
186.             context.EmitKeyValue(key, value.ToString());
187.
188.         }
189.
190.     }
191.
192.     public class MySimpleReducer : ReducerCombinerBase
193.     {
194.
195.         public override void Reduce(
196.
197.             string key, IEnumerable values, ReducerCombinerContext context
198.
199.             )
200.         {
201.
202.             //initialize counters
203.
204.             int myCount = 0;
205.
206.             int mySum = 0;
207.
208.
209.
210.             //count and sum incoming values
211.
212.             foreach (string value in values)
213.             {
214.
215.                 mySum += int.Parse(value);
216.
217.                 myCount++;
218.
219.             }
220.
221.
222.
223.             //output results
224.
225.             context.EmitKeyValue(key, myCount + "\t" + mySum);
226.
227.         }
228.
229.
230.     }
231. }

三、测试结果

大数据系列相关文章:

最新评论
用心去把握2014-09-10 11:40:34
你造卢辉撒
ONE2014-09-10 03:07:22
威尔逊2014-09-09 04:41:00
搜索引擎要做分词
李丽2014-09-09 03:12:32
翻来干嘛
阿神hzl2014-09-07 11:00:23
vnc
 
  • Hadoop生态系统资料推荐