[ARCH,PATCHv3] ipc design and usage modes

diff --git a/ipc.dox b/ipc.dox
new file mode 100644
index 0000000..ab85011
--- /dev/null
+++ b/ipc.dox
@@ -0,0 +1,198 @@ 
+/* Copyright (c) 2014, Linaro Limited
+ * All rights reserved
+ *
+ * SPDX-License-Identifier:     BSD-3-Clause
+ */
+
+/**
+@page ipc_design Inter Process Communication (IPC) API
+
+@tableofcontents
+
+@section ipc_intro Introduction
+	This document defines the two different ODP application modes
+	multithreading and multiprocessing with respect to their impact on IPC
+
+@subsection odp_modes Application Thread/Process modes:
+	ODP applications can use following programming models for multi core support:
+	-# Single application with ODP worker Threads.
+	-# Multi process application with single packet I/O pool and common initialization.
+	-# Different processed communicated thought IPC API.
+
+@todo - add diagram about IPC modes.
+
+@subsubsection odp_mode_threads Thread mode
+	The initialization sequence for thread mode is following:
+
+@verbatim
+	 main() {
+		/* Init ODP before calling anything else. */
+		odp_init_global(NULL, NULL);
+
+		/* Init this thread. */
+		odp_init_local();
+
+		/* Allocate memory for packets pool. That memory will be visible for all threads.*/
+		shm = odp_shm_reserve("shm_packet_pool",
+				      SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0);
+		pool_base = odp_shm_addr(shm);
+
+		/* Create pool instance with reserved shm. */
+		pool = odp_buffer_pool_create("packet_pool", pool_base,
+				      SHM_PKT_POOL_SIZE,
+				      SHM_PKT_POOL_BUF_SIZE,
+				      ODP_CACHE_LINE_SIZE,
+				      ODP_BUFFER_TYPE_PACKET);
+
+		/* Create worker threads. */
+		odph_linux_pthread_create(&thread_tbl[i], 1, core, thr_run_func,
+					  &args);
+		}
+
+	/* thread function */
+	thr_run_func () {
+		/* Lookup the packet pool */
+		pkt_pool = odp_buffer_pool_lookup("packet_pool");
+
+		/* Open a packet IO instance for this thread */
+		pktio = odp_pktio_open("eth0", pkt_pool);
+
+		for (;;) {
+			/* read buffer */
+			buf = odp_schedule(NULL, ODP_SCHED_WAIT);
+			... do something ...
+		}
+	}
+@endverbatim
+
+@subsubsection odp_mode_processes Processes mode with shared memory
+	Initialization sequence in processes mode with shared memory is following:
+
+@verbatim
+	 main() {
+		 /* Init ODP before calling anything else. In process mode odp_init_global
+		  * function called only once in main run process.
+		  */
+		 odp_init_global(NULL, NULL);
+
+		 /* Init this thread. */
+		 odp_init_local();
+
+		 /* Allocate memory for packets pool. That memory will be visible for all threads.*/
+		 shm = odp_shm_reserve("shm_packet_pool",
+				 SHM_PKT_POOL_SIZE, ODP_CACHE_LINE_SIZE, 0);
+		 pool_base = odp_shm_addr(shm);
+
+		 /* Create pool instance with reserved shm. */
+		 pool = odp_buffer_pool_create("packet_pool", pool_base,
+				 SHM_PKT_POOL_SIZE,
+				 SHM_PKT_POOL_BUF_SIZE,
+				 ODP_CACHE_LINE_SIZE,
+				 ODP_BUFFER_TYPE_PACKET);
+
+		 /* Call odph_linux_process_fork_n which will fork() current process to
+		  * different processes.
+		  */
+		 odph_linux_process_fork_n(proc, num_workers, first_core);
+
+		 /* Run same function as thread uses */
+		 thr_run_func();
+		}
+
+	/* thread function */
+	thr_run_func () {
+		/* Lookup the packet pool */
+		pkt_pool = odp_buffer_pool_lookup("packet_pool");
+
+		/* Open a packet IO instance for this thread */
+		pktio = odp_pktio_open("eth0", pkt_pool);
+
+		for (;;) {
+			/* read buffer */
+			buf = odp_schedule(NULL, ODP_SCHED_WAIT);
+			... do something ...
+		}
+	}
+@endverbatim
+
+@subsubsection odp_mode_sep_processes Separate Processes mode
+	This mode differs from mode with common shared memory. Each execution thread is completely independent process which calls
+	odp_init_global() and do other initialization process then opens IPC pktio interface and does packets exchange between processes
+	to communicate between these independent processes. IPC pktio interface may be used to exchange packets.
+	For the base implementation (linux-generic) shared memory is used as the IPC mechanism to make it easy to reuse for different use
+	cases. The use cases are process that may be spread amongst different VMs, bare metal or regular Linux user space, in fact any
+	process that can share memory.
+
+	In hardware implementations IPC pktio can be offloaded to HW SoC packets functions.
+	The initialization sequence in the separate thread mode model is same as it is process, both using shared memory but with following
+	difference:
+
+@subsubsection odp_mode_sep_processes_cons Separate Processes Sender (linux-generic)
+	-#  Each process calls odp_init_global(), pool creation and etc.
+
+	-# ODP_SHM_PROC flag provided to be able to map that memory from different process.
+
+@verbatim
+        shm = odp_shm_reserve("shm_packet_pool",
+                              SHM_PKT_POOL_SIZE,
+                              ODP_CACHE_LINE_SIZE,
+                              ODP_SHM_PROC);
+
+        pool_base = odp_shm_addr(shm);
+@endverbatim
+
+	-# Worker thread (or process) creates IPC pktio, and sends buffers to it:
+
+	A)
+	odp_pktio_t ipc_pktio = odp_pktio_open("ipc_pktio", pkt_pool);
+	odp_pktio_send(ipc_pktio, pkt_tbl, pkts);
+
+	B) instead of using packet io queue can be used in following way:
+
+@verbatim
+	odp_queue_t ipcq = odp_pktio_outq_getdef(ipc_pktio);
+	/* Then enqueue the packet for output queue */
+	odp_queue_enq(ipcq, buf);
+@endverbatim
+
+@subsubsection odp_mode_sep_processes_recv Separate Processes Receiver (linux-generic)
+	On the other end process also creates IPC packet I/O and receives packets
+	from it.
+
+@verbatim
+
+	/* Do lookup packet I/O in IPC shared memory,
+	 * and link it to local pool. */
+	while (1) {
+		pktio = odp_pktio_lookup("ipc_pktio");
+		if (pktio == ODP_PKTIO_INVALID) {
+			sleep(1);
+			printf("pid %d: looking for ipc_pktio\n", getpid());
+			continue;
+		}
+		break;
+	}
+
+	/* Get packets from the IPC */
+	for (;;) {
+		pkts = odp_pktio_recv(pktio, pkt_tbl, MAX_PKT_BURST);
+		...
+	}
+@endverbatim
+
+@subsubsection odp_mode_sep_processes_hw Separate Processes Hardware optimized
+	Hardware SoC implementation of IPC exchange can differ. It can use a shared pool
+	or it can rely on the hardware for packet transmission. But the API interface remains the
+
+
+	Hardware SoC implementation of IPC exchange can differ. It can use share pool
+	or can relay on hardware for packet transmission. But the API interface remains the
+	same:
+
+	odp_pktio_open(), odp_pktio_lookup()
+
+@todo - Bug 825 odp_buffer_pool_create() API will change to allocate memory for pool inside it.
+	So odp_shm_reserve() for remote pool memory and odp_pktio_lookup() can go inside
+	odp_buffer_pool_create().
+
+*/