#! /usr/bin/perl
# 
# $Id$
# Captive project doc APITypes page Perl template.
# Copyright (C) 2003 Jan Kratochvil <project-www.jankratochvil.net@jankratochvil.net>
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; exactly version 2 of June 1991 is required
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA


package project::captive::doc::APITypes;
require 5.6.0;	# at least 'use warnings;' but we need some 5.6.0+ modules anyway
our $VERSION=do { my @r=(q$Revision$=~/\d+/g); sprintf "%d.".("%03d"x$#r),@r; };
our $CVS_ID=q$Id$;
use strict;
use warnings;

BEGIN{ open F,"Makefile"; our $top_dir=pop @{[split /\s/,(grep /^top_srcdir/,<F>)[0]]}; eval "use lib '$top_dir'"; close F; }
use My::Web;
require CGI;
BEGIN { Wuse 'project::captive::doc::Macros'; }


My::Web->init(
		"__PACKAGE__"=>__PACKAGE__,
		"title"=>'Captive NTFS Developer Documentation: API Functions',
		"head_css"=>$doc_Macros_head_css,
		);
My::Web->heading();


print <<"HERE";


<a name="functype"><h1>API Function Implementation Choices</h1></a>

	<p>For each function exported by W32
	<span class="fname">ntoskrnl.exe</span> and imported and called by the
	filesystem driver a decision needs to be made to properly implement its
	functionality. Currently implemented functionality statistics are provided
	below:</p>

	<table border="1" align="center">
		<tr><th>Function type                                            </th><th>Items</th><th>Portion</th></tr>
		<tr><td>@{[ a_href 'APITypes.html.pl#functype_pass','pass' ]}                    </td><td>   81</td><td>    26%</td></tr>
		<tr><td>@{[ a_href 'APITypes.html.pl#functype_wrap','wrap' ]}                    </td><td>    2</td><td>     0%</td></tr>
		<tr><td>@{[ a_href 'APITypes.html.pl#functype_native_reactos','native-ReactOS' ]}</td><td>  113</td><td>    36%</td></tr>
		<tr><td>@{[ a_href 'APITypes.html.pl#functype_native_libcaptive','native-own' ]} </td><td>  116</td><td>    38%</td></tr>
		<caption>Function Implementation Types Statistics</caption>
	</table>

	@{[ doc_img 'ratio','Functions Reusal Ratio' ]}

	<p>As there are several choices to implement each function the usual
	attempts/investigations ordering is listed in the sections below.</p>

	<p>Special case must be taken for data-type symbols since they are
	referenced without the possibility of catching the code flow by some
	breakpoints (it would be possible only in some special access cases). Data
	export symbols of <span class="constant">unpatched</span> libraries must
	contain already prepared content at the runtime. There is a&nbsp;problem
	with <span class="constant">patched</span> libraries where it is necessary
	to also fully implement the data symbol as
	@{[ a_href 'APITypes.html.pl#functype_native','native implementation' ]} since there is no
	possibility to @{[ a_href 'APITypes.html.pl#functype_pass','pass' ]} the data symbol instead of
	the original W32 data location and therefore there will be two instances of
	such data variable place. As there will be also the uncaught references for
	such W32 data location from the <span class="constant">patched</span>
	library itself such symbols should be usually only some constants (such as
	<span class="constant">KeNumberProcessors</span>).</p>

	<p>W32 platform symbols export/import can be based either on the symbol
	name itself or it can be also exported and imported just by its
	identification number called <span class="constant">Ordinal</span>.
	Although it saves some jumptables file binary size it is currently no
	longer used by W32 binaries and this project also does not support such
	<span class="constant">Ordinal</span> symbol reference type at all.</p>

	<p>All the exporting magic is handled by custom script
	<span class="fname">captivesym</span> processing the definition file
	<span class="fname">@{[ captive_srcfile 'src/libcaptive/ke/exports.captivesym' ]}</span>
	to produce the intermediate relaying code
	<span class="fname">src/libcaptive/ke/exports.c</span>. For details of the
	<span class="fname">captivesym</span>-specific source file syntax please
	see its documentation:
	<span class="fname">@{[ a_href
			'/project/Pod2Html.html.pl?cvs=priv/captive/src/libcaptive/ke/captivesym.pl',
			'src/libcaptive/ke/captivesym.pl' ]}</span>

	<a name="functype_pass"><h2>Direct Pass to Original &quot;ntoskrnl.exe&quot;</h2></a>

		<p>Simple (standalone) functions such as
		<span class="function">RtlTimeToSecondsSince1970()</span> can be simply
		passed to the original implementation in
		<span class="fname">ntoskrnl.exe</span> as they make no hardware access
		and they do not expect any special internal data structures to be set up
		in advance by an earlier library initialization. A common case are all
		the data structures utility functions such as
		<span class="constant">GenericTable</span> subsystem or
		<span class="constant">LargeMcb</span> handling.</p>

		<a name="functype_pass_fromunix"><h3>Pass from UNIX Code</h3></a>

			<p>Control flow begins in some standard UNIX code. Such code is always
			using @{[ a_href 'CallType.html.pl#calltype_cdecl','cdecl call type' ]} for all its
			intracalls. <a href="APITypes.html.pl#functype_native_reactos">Native functions
			compiled from <span class="productname">ReactOS</span> sources</a> use
			their own @{[ a_href '#calltype','cdecl/stdcall/fastcall' ]} declarations
			but these call type modifications are discarded during compilation for
			this project by the <span class="constant">LIBCAPTIVE</span>
			symbol.</p>

			<p>UNIX code calls <span class="function">FUNCTIONNAME()</span> relay
			from the generated UNIX jump table. Such relay will debug dump the
			passed arguments and finally pass the control to the original W32
			function code in the proper call type
			@{[ a_href '#calltype','cdecl/stdcall/fastcall' ]} for a&nbsp;given
			function.</p>

			<p>Original W32 code entry point is always trapped by a&nbsp;breakpoint
			although it would not be needed during this specific direct pass from
			UNIX code to the original W32 implementation. Still the breakpoint has
			to be there to catch some other (such as intra-W32) possible calls
			described later. There are several more ways to define breakpoint in
			the code. One way is to use processor hardware breakpoint support but
			the number of breakpoints is limited.  The other way is to patch in the
			<span class="instruction">@{[ 'int $3' ]}</span> instruction but it will invoke
			<span class="constant">SIGTRAP</span> signal handler conflicting with
			the possible debugger (<span class="productname">gdb(1)</span>)
			control. This project uses the <span class="instruction">hlt</span>
			instruction, which also has a&nbsp;single-byte opcode as
			<span class="instruction">@{[ 'int $3' ]}</span> and it is a&nbsp;privileged
			instruction forbidden to be used from the UNIX user space code.
			<span class="instruction">hlt</span> invokes
			<span class="constant">SIGSEGV</span> signal which can be resolved by
			a&nbsp;custom signal handler without any conflict with the possible
			debugger control; <span class="productname">gdb(1)</span> needs the
			following command to pass through such
			<span class="constant">SIGSEGV</span> signal:</p>

			<blockquote class="command">
				<p>handle SIGSEGV nostop noprint pass</p>
			</blockquote>

			<p>When a breakpoint gets caught, we usually need to return to the
			running code. Unfortunately it is not possible because of the patched
			breakpoint opcode. The breakpoint cannot be simply removed upon return
			as it would permanently loose control over the point of entry. Even if
			the return would include faking of the return address in the bottom
			stack frame to patch the breakpoint back during later function exit it
			still would not solve the caughts of inner calls of recursive
			functions. One of the working possibilities would be to patch the
			original instruction back and perform a&nbsp;singlestep provided by
			<span class="function">ptrace(2)</span> syscall. However such
			singlestep needs another controlling UNIX process and it would again
			conflict with the debuggers such as
			<span class="productname">gdb(1)</span>. This project implements the
			singlestep functionality by two consecutive breakpoints
			(<span class="instruction">hlt</span> instructions to be specific):
			The first two instruction addresses of the W32 functions are called
			<span class="productname">slot #1</span> and
			<span class="productname">slot #2</span>, the length of the first
			function instruction has to be analyzed to get the right address of
			<span class="productname">slot #2</span>. When the first breakpoint is
			caught it is necessary to patch the original instruction back and also
			patch another breakpoint in place of
			<span class="productname">slot #2</span>.
			During the <span class="productname">slot #2</span> breakpoint
			invocation the operation will be reverted &mdash; the breakpoint will be put
			to <span class="productname">slot #1</span> again and the instruction
			of <span class="productname">slot #2</span> will be restored to be able
			to continue the execution of the function.</p>

			<p>W32 function will finish in its specific
			@{[ a_href '#calltype','cdecl/stdcall/fastcall call type' ]}, the control
			will return to the UNIX jump table relay which will debug dump the
			return value and it will finally pass the control back to the UNIX
			caller in the standard UNIX
			@{[ a_href 'CallType.html.pl#calltype_cdecl','cdecl call type' ]}.</p>

			@{[ doc_img 'fig/functype_patched_pass_fromunix',
					'Function Type: <span class="constant">pass</span> from UNIX Code' ]}

		<a name="functype_pass_fromw32"><h3>Pass from W32 Code</h3></a>

			<p>This function type is similiar to the
			@{[ a_href 'APITypes.html.pl#functype_pass_fromunix','previous one' ]} with the exception
			of more complicated entry point. Unfortunately W32 libraries call their
			own functions directly, using the <span class="instruction">call</span>
			instructions without any patchable jump table. Even the
			<span class="instruction">call</span> argument itself cannot be patched
			according to the relocation table record as such library intra-call
			instruction has no relocation due to its relative argument offset on
			<span class="constant">i386</span>. This time the double-breakpoint
			mechanism @{[ a_href 'APITypes.html.pl#functype_pass_fromunix','described above' ]} gets
			handy since it will catch the entry point when the function gets
			called.  <span class="constant">SIGSEGV</span> handler gets invoked by
			the <span class="instruction">hlt</span> instruction and it will
			redirect the control to the jump table relay function to debug dump the
			function entry arguments (it has no other uses in this call type).</p>

			<p>When the relay needs to call the original function it will reach
			exactly the same breakpoint instruction as during the recent
			<span class="constant">SIGSEGV</span> handling redirecting to this
			calling relay.  But this time the
			<span class="constant">through_w32_func</span> field of this function
			record will be set to to prevent repeated redirection and to pass the
			control through the breakpoint mangle instead this time.</p>

			<p>Returning is not much interesting as the first
			<span class="constant">SIGSEGV</span> handler did a&nbsp;straight jump
			for the redirection purposes without any needed consequent
			handling.</p>

			<p>The jump table relay used for the callers from W32 code is
			a&nbsp;different one than the relay being used for the callers
			@{[ a_href 'APITypes.html.pl#functype_pass_fromunix','from UNIX code' ]}. UNIX code always
			uses relay with external @{[ a_href 'CallType.html.pl#calltype_cdecl','cdecl call type' ]}
			but in this case a&nbsp;relay with the appropriate
			@{[ a_href '#calltype','cdecl/stdcall/fastcall call type' ]} is used.</p>

			@{[ doc_img 'fig/functype_patched_pass_fromw32',
					'Function Type: <span class="constant">pass</span> from W32 Code' ]}

		@{[ vskip() ]}

		<table border="1" align="center">
			<tr><td><span class="fname">captivesym</span> keyword</td><td>pass</td></tr>
			<tr><td>Native code function name                    </td><td>(no implementation)</td></tr>
			<tr><td>W32 traced code from UNIX function name      </td><td>FUNCNAME</td></tr>
			<tr><td>W32 traced code from W32  function name      </td><td>FUNCNAME_cdecl/_stdcall/_fastcall</td></tr>
			<tr><td>Entry/exit debug tracing from UNIX code      </td><td>yes</td></tr>
			<tr><td>Entry/exit debug tracing from W32 code       </td><td>yes</td></tr>
			<caption>Function Type <span class="constant">pass</span> Characteristics</caption>
		</table>

	<a name="functype_wrap"><h2>Wrap of the Original "ntoskrnl.exe" Function</h2></a>

		<a name="functype_wrap_fromunix"><h3>Wrapping of Call from UNIX Code</h3></a>

			<p>The code control flow has no special hardcore features since it is
			very similiar to <a href="APITypes.html.pl#functype_pass_fromunix">the direct pass to
			W32 function from UNIX code</a>. All the wrapping is done in the
			standard UNIX @{[ a_href 'CallType.html.pl#calltype_cdecl','cdecl call type' ]} manner.
			Jump table debug dumping relays are provided twice &mdash; the
			&quot;outer&quot; one to trace the parameters from the function caller
			and the &quot;inner&quot; one to trace the call from the wrapper to the
			original W32 code. The &quot;inner&quot; relay also calls the W32 code
			with the appropriate <a href="#calltype">cdecl/stdcall/fastcall call
			type</a>.</p>

			@{[ doc_img 'fig/functype_patched_wrap_fromunix',
					'Function Type: <span class="constant">wrap</span> from UNIX Code' ]}

		<a name="functype_wrap_fromw32"><h3>Wrapping of Call from W32 Code</h3></a>

			<p>This scheme is a&nbsp;combination of the
			<a href="APITypes.html.pl#functype_wrap_fromunix">previous wrap of a&nbsp;call from
			UNIX code</a> and the <a href="APITypes.html.pl#functype_pass_fromw32">direct pass from
			the W32 code</a>. The control is caught and redirected by
			<span class="constant">SIGSEGV</span> handler from the breakpoint
			placed at the entry to the original W32 function code. The second entry
			to the original W32 function with the
			<span class="constant">through_w32_func</span> field of this function
			description already set is done from the &quot;inner&quot; jump table
			relay with the appropriate
			@{[ a_href '#calltype','cdecl/stdcall/fastcall call type' ]}.</p>

			@{[ doc_img 'fig/functype_patched_wrap_fromw32',
					'Function Type: <span class="constant">wrap</span> from W32 Code' ]}

		@{[ vskip() ]}

		<p>Some functions can be <a href="APITypes.html.pl#functype_pass">passed to the original
		code</a> but they need their parameters to be checked/prepared.
		Currently, such wrapping is only needed for the
		<span class="function">ExAllocateFromPagedLookasideList()</span> function
		where it is required due to <a href="#init_ntoskrnl">missing execution of
		<span class="fname">ntoskrnl.exe</span> initialization execution</a>,
		which would otherwise properly initialize some internal data structures.
		In this case the wrapping code detects passing of an uninitialized
		parameter and will search through the whole
		<span class="fname">ntoskrnl.exe</span> code body at runtime to find the
		proper initialization routine containing the correct initialization
		parameters.  Passed addresses of static structures must be differentiated
		as each of them usually has different initialization parameters. It is
		proactive to not to have fixed parameters array as these parameters may
		differ across different <span class="fname">ntoskrnl.exe</span>
		versions.</p>

		<table border="1" align="center">
			<tr><td><span class="fname">captivesym</span> keyword</td><td>wrap</td></tr>
			<tr><td>Native UNIX wrapping code function name      </td><td>FUNCNAME_wrap</td></tr>
			<tr><td>W32 traced wraping code from UNIX func. name </td><td>FUNCNAME</td></tr>
			<tr><td>W32 traced wrapping code from W32 func. name </td><td>FUNCNAME_cdecl/_stdcall/...</td></tr>
			<tr><td>W32 traced original code function name       </td><td>FUNCNAME_orig</td></tr>
			<tr><td>Entry/exit debug tracing from UNIX code      </td><td>yes</td></tr>
			<tr><td>Entry/exit debug tracing from W32 code       </td><td>yes</td></tr>
			<caption>Function Type <span class="constant">wrap</span> Characteristics</caption>
		</table>

	<a name="functype_native"><h2>Native Implementation</h2></a>

		<a name="functype_native_fromunix"><h3>Native Implementation Called from UNIX Code</h3></a>

			<p>This is the simplest case of a&nbsp;function call as it is fully
			handled only by the compiler and/or linker.</p>

			<p>In this case though, no debug dumping call relay is provided &mdash; such
			relay would need to rename the implementations of native functions to
			prevent its automatic linking with the caller code. This renaming would
			not be possible to do by simple <span class="constant">#define</span>
			since it would also rename any calling statements of such function in
			the same C&nbsp;sources.  One of the possibilities to solve would be to
			utilize <span class="dashdash">--redefine-sym</span> feature of the
			<span class="productname">objcopy(1)</span> utility. On the other hand
			there is not much need to catch/debug such calls as both the caller and
			the callee are provided with full source file debug information for the
			debugger. Also the callee usually debug dumps its entry/exit parameters
			by custom debug dumps in the
			<a href="APITypes.html.pl#functype_native_reactos"><span class="productname">ReactOS</span> implementations</a>.

			@{[ doc_img 'fig/functype_native_fromunix',
					'Function Type: <span class="constant">native</span> from UNIX Code' ]}

		<a name="functype_native_fromw32"><h3>Native Implementation of
				&quot;unpatched&quot; Library Function Called from W32 Code</h3></a>

			@{[ doc_img 'fig/functype_unpatched_native_fromw32',
					'Function Type: <span class="constant">native</span> of <span class="constant">unpatched</span> from W32 Code' ]}

			<p>Here comes the differentiation if the project deals either with
			a&nbsp;<span class="constant">patched</span> or an
			<span class="constant">unpatched</span> version of the library
			(<span class="constant">patched</span> is a&nbsp;loaded W32 binary
			library while <span class="constant">unpatched</span> library is
			completely provided by this project with no use of the library's
			original W32 binary file). As the project adjusts the exported symbol
			address during the patching operation, in some cases the
			<span class="constant">patched</span> library call may be handled
			simply as <span class="constant">unpatched</span> library call even for
			the <span class="constant">patched</span> libraries. Fortunately the
			distinction is not much important as the project is prepared to
			properly handle both cases.</p>

			<p>The W32 caller which imported the symbol will be pointed right to
			the relaying function. The debug dumping relay will be called from W32
			code with the appropriate
			@{[ a_href '#calltype','cdecl/stdcall/fastcall call type' ]} while the
			relay will call the implementation of the native function in the
			standard UNIX @{[ a_href 'CallType.html.pl#calltype_cdecl','cdecl call type' ]} manner.</p>

		<a name="functype_native_fromw32_patched"><h3>Native Implementation of &quot;patched&quot; Library Function Called from W32 Code</h3></a>

			@{[ doc_img 'fig/functype_patched_native_fromw32',
					'Function Type: <span class="constant">native</span> of <span class="constant">patched</span> from W32 Code' ]}

			<p>The calling scheme is similiar to the
			<a href="APITypes.html.pl#functype_native_fromw32">previous call of
			<span class="constant">unpatched</span> library function from W32
			code</a> but the call control is redirected from the entry point of the
			original W32 binary implementation by the breakpoint and its
			<span class="constant">SIGSEGV</span> handler as in
			<a href="APITypes.html.pl#functype_pass_fromw32">the case of passing control from W32
			call</a>.</p>

			<p>The original W32 function implementation located in the original
			loaded binary file is never executed but its entry point needs to be
			trapped by the breakpoint to be able to catch the function calls within
			the library.</p>

		@{[ vskip() ]}

		<p>In all cases the final function implementation is a&nbsp;standard UNIX
		code compiled from C&nbsp;sources with full debug information available
		for the debugger. Fortunately all such functions do not need to be coded
		from scratch for this project since there already exist $freespeech
		$ReactOS and $Wine projects and their code can be used instead.</p>

		<p>$Wine project is listed mostly for a&nbsp;completeness as almost no
		code was suitable for reuse as it implements W32 user space while this
		project is running pure W32 kernel space environment (in $gnulinux user
		space!).</p>

		<a name="functype_native_reactos"><h3>Native Implementation
				- <span class="productname">ReactOS</span></h3></a>

			<p>Some functions are already implemented in the $ReactOS
			project and they can be used as they are.  Although it would be
			possible to <a href="APITypes.html.pl#functype_pass">pass some function calls to the
			original code</a> it is more handy to provide native implementation as
			there is better control of the data handling during debugging sessions
			due to the provided debugging symbols.</p>

			<p>Such functions can be found in
			<span class="fname">src/libcaptive/reactos/</span> subdirectory.
			Some functions had to be adjusted for this project
			- these modifications are compiled conditionally, depending on the
			<span class="constant">LIBCAPTIVE</span> symbol existence.</p>

			<p>Later stages of this project reached the level where
			$ReactOS is yet too immature and the needed functions are usually
			written just with the sad body:</p>

			<blockquote class="command">
				<p>UNIMPLEMENTED;</p>
			</blockquote>

			<p>Functions that were not possible to
			@{[ a_href 'APITypes.html.pl#functype_pass','pass' ]} were reimplemented by this project
			and placed in the project's implementation directories
			@{[ a_href '#reactos_nocare','instead of extending' ]} $ReactOS code.</p>

		<a name="functype_native_wine"><h3>Native Implementation &ndash; <span class="productname">Wine</span></h3></a>

			<p>Even though $Wine only implements the
			<span class="productname">Microsoft Windows NT</span> user space, there
			still are some common functions which could be copied from the $Wine
			project.</p>

		<a name="functype_native_libcaptive"><h3>Native Implementation &ndash; Project Specific</h3></a>

			<p>As the last resort it was necessary to provide completely own
			implementation of some API functions such as PC hardware dependent
			parts or memory management functions.</p>

		@{[ vskip() ]}

		<table border="1" align="center">
			<tr><td><span class="fname">captivesym</span> keyword</td><td>(none; just the symbol name)</td></tr>
			<tr><td>Native code function name                    </td><td>FUNCTIONNAME</td></tr>
			<tr><td>Native traced code from W32 code func. name  </td><td>FUNCTIONNAME_cdecl/_std...</td></tr>
			<tr><td>Entry/exit debug tracing from UNIX code      </td><td>no</td></tr>
			<tr><td>Entry/exit debug tracing from W32 code       </td><td>yes</td></tr>
			<caption>Function Type <span class="constant">native</span> Characteristics</caption>
		</table>

	<a name="functype_undef"><h2>Undefined Function</h2></a>

		<p>Functions not defined by any of the previous function types cannot be
		called by any W32 code including the code of the library implementing
		such function. All functions of <span class="constant">patch</span>ed
		libraries not listed in the <span class="fname">captivesym</span> exports
		file are automatically set to be trapped as fatal program execution
		errors.</p>

		<p>It is not necessary to list the symbols as
		<span class="constant">undef</span> as long as you are just loading the
		W32 <span class="constant">PE-32</span> code and the symbols belong to
		<span class="constant">patch</span>ed library. On the other hand if you
		are loading W32 <span class="fname">.so</span> code or if such symbol is
		a&nbsp;part of <span class="constant">unpatched</span> library (and thus
		being completely provided by the project) you need to list such symbol as
		<span class="constant">undef</span> type to prevent unresolved symbol
		reference.</p>

		<table border="1" align="center">
			<tr><td><span class="fname">captivesym</span> keyword</td><td>undef</td></tr>
			<tr><td>Native code function name                    </td><td>(no implementation)</td></tr>
			<tr><td>Native traced code function name             </td><td>FUNCTIONNAME_cdecl/_stdcall/_fastcall</td></tr>
			<tr><td>Debug tracing message from UNIX code         </td><td>yes</td></tr>
			<tr><td>Debug tracing message from W32 code          </td><td>yes</td></tr>
			<caption>Function Type <span class="constant">undef</span> Characteristics</caption>
		</table>

	
HERE


My::Web->footer();
