Code:
/ Dotnetfx_Vista_SP2 / Dotnetfx_Vista_SP2 / 8.0.50727.4016 / DEVDIV / depot / DevDiv / releases / Orcas / QFE / wpf / src / Shared / MS / Internal / TokenizerHelper.cs / 1 / TokenizerHelper.cs
//---------------------------------------------------------------------------- // // Copyright (C) Microsoft Corporation. All rights reserved. // // File: TokenizerHelper.cs // // Description: This file contains the implementation of TokenizerHelper. // This class should be used by most - if not all - MIL parsers. // // History: // 05/19/2003 : adsmith - Created it // 05/20/2003 : adsmith - Moved to Shared // //--------------------------------------------------------------------------- using System; using System.Collections; using System.Diagnostics; using System.ComponentModel; using System.Globalization; #if !PBTCOMPILER using System.Windows; #endif #if WINDOWS_BASE using MS.Internal.WindowsBase; #elif PRESENTATION_CORE using MS.Internal.PresentationCore; #elif PRESENTATIONFRAMEWORK using MS.Internal.PresentationFramework; #elif PBTCOMPILER using MS.Utility ; using MS.Internal.Markup; #elif DRT using MS.Internal.Drt; #else #error Attempt to use FriendAccessAllowedAttribute from an unknown assembly. using MS.Internal.YourAssemblyName; #endif #if PBTCOMPILER namespace MS.Internal.Markup #else namespace MS.Internal #endif { #if !PBTCOMPILER [FriendAccessAllowed] #endif internal class TokenizerHelper { ////// Constructor for TokenizerHelper which accepts an IFormatProvider. /// If the IFormatProvider is null, we use the thread's IFormatProvider info. /// We will use ',' as the list separator, unless it's the same as the /// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one /// number or two. In this case, we will use ";" as the separator. /// /// The string which will be tokenized. /// The IFormatProvider which controls this tokenization. internal TokenizerHelper(string str, IFormatProvider formatProvider) { char numberSeparator = GetNumericListSeparator(formatProvider); Initialize(str, '\'', numberSeparator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. internal TokenizerHelper(string str, char quoteChar, char separator) { Initialize(str, quoteChar, separator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. private void Initialize(string str, char quoteChar, char separator) { _str = str; _strLen = str == null ? 0 : str.Length; _currentTokenIndex = -1; _quoteChar = quoteChar; _argSeparator = separator; // immediately forward past any whitespace so // NextToken() logic always starts on the first // character of the next token. while (_charIndex < _strLen) { if (!Char.IsWhiteSpace(_str, _charIndex)) { break; } ++_charIndex; } } internal string GetCurrentToken() { // if no current token, return null if (_currentTokenIndex < 0) { return null; } return _str.Substring(_currentTokenIndex,_currentTokenLength); } ////// Throws an exception if there is any non-whitespace left un-parsed. /// internal void LastTokenRequired() { if (_charIndex != _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken() { return NextToken(false); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired() { if (!NextToken(false)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired(bool allowQuotedToken) { if (!NextToken(allowQuotedToken)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken) { // use the currently-set separator character. return NextToken(allowQuotedToken, _argSeparator); } ////// Advances to the NextToken. A separator character can be specified /// which overrides the one previously set. /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken, char separator) { _currentTokenIndex = -1; // reset the currentTokenIndex _foundSeparator = false; // reset // If we're at end of the string, just return false. if (_charIndex >= _strLen) { return false; } char currentChar = _str[_charIndex]; Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace"); // setup the quoteCount int quoteCount = 0; // If we are allowing a quoted token and this token begins with a quote, // set up the quote count and skip the initial quote if (allowQuotedToken && currentChar == _quoteChar) { quoteCount++; // increment quote count ++_charIndex; // move to next character } int newTokenIndex = _charIndex; int newTokenLength = 0; // loop until hit end of string or hit a , or whitespace // if at end of string ust return false. while (_charIndex < _strLen) { currentChar = _str[_charIndex]; // if have a QuoteCount and this is a quote // decrement the quoteCount if (quoteCount > 0) { // if anything but a quoteChar we move on if (currentChar == _quoteChar) { --quoteCount; // if at zero which it always should for now // break out of the loop if (0 == quoteCount) { ++_charIndex; // move past the quote break; } } } else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator)) { if (currentChar == separator) { _foundSeparator = true; } break; } ++_charIndex; ++newTokenLength; } // if quoteCount isn't zero we hit the end of the string // before the ending quote if (quoteCount > 0) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote)); } ScanToNextToken(separator); // move so at the start of the nextToken for next call // finally made it, update the _currentToken values _currentTokenIndex = newTokenIndex; _currentTokenLength = newTokenLength; if (_currentTokenLength < 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } return true; } // helper to move the _charIndex to the next token or to the end of the string void ScanToNextToken(char separator) { // if already at end of the string don't bother if (_charIndex < _strLen) { char currentChar = _str[_charIndex]; // check that the currentChar is a space or the separator. If not // we have an error. this can happen in the quote case // that the char after the quotes string isn't a char. if (!(currentChar == separator) && !Char.IsWhiteSpace(currentChar)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } // loop until hit a character that isn't // an argument separator or whitespace. // !!!Todo: if more than one argSet throw an exception int argSepCount = 0; while (_charIndex < _strLen) { currentChar = _str[_charIndex]; if (currentChar == separator) { _foundSeparator = true; ++argSepCount; _charIndex++; if (argSepCount > 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } else if (Char.IsWhiteSpace(currentChar)) { ++_charIndex; } else { break; } } // if there was a separatorChar then we shouldn't be // at the end of string or means there was a separator // but there isn't an arg if (argSepCount > 0 && _charIndex >= _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } } // Helper to get the numeric list separator for a given IFormatProvider. // Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise. static internal char GetNumericListSeparator(IFormatProvider provider) { char numericSeparator = ','; // Get the NumberFormatInfo out of the provider, if possible // If the IFormatProvider doesn't not contain a NumberFormatInfo, then // this method returns the current culture's NumberFormatInfo. NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider); Debug.Assert(null != numberFormat); // Is the decimal separator is the same as the list separator? // If so, we use the ";". if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0])) { numericSeparator = ';'; } return numericSeparator; } internal bool FoundSeparator { get { return _foundSeparator; } } char _quoteChar; char _argSeparator; string _str; int _strLen; int _charIndex; internal int _currentTokenIndex; internal int _currentTokenLength; bool _foundSeparator; } } // File provided for Reference Use Only by Microsoft Corporation (c) 2007. // Copyright (c) Microsoft Corporation. All rights reserved. //---------------------------------------------------------------------------- // // Copyright (C) Microsoft Corporation. All rights reserved. // // File: TokenizerHelper.cs // // Description: This file contains the implementation of TokenizerHelper. // This class should be used by most - if not all - MIL parsers. // // History: // 05/19/2003 : adsmith - Created it // 05/20/2003 : adsmith - Moved to Shared // //--------------------------------------------------------------------------- using System; using System.Collections; using System.Diagnostics; using System.ComponentModel; using System.Globalization; #if !PBTCOMPILER using System.Windows; #endif #if WINDOWS_BASE using MS.Internal.WindowsBase; #elif PRESENTATION_CORE using MS.Internal.PresentationCore; #elif PRESENTATIONFRAMEWORK using MS.Internal.PresentationFramework; #elif PBTCOMPILER using MS.Utility ; using MS.Internal.Markup; #elif DRT using MS.Internal.Drt; #else #error Attempt to use FriendAccessAllowedAttribute from an unknown assembly. using MS.Internal.YourAssemblyName; #endif #if PBTCOMPILER namespace MS.Internal.Markup #else namespace MS.Internal #endif { #if !PBTCOMPILER [FriendAccessAllowed] #endif internal class TokenizerHelper { ////// Constructor for TokenizerHelper which accepts an IFormatProvider. /// If the IFormatProvider is null, we use the thread's IFormatProvider info. /// We will use ',' as the list separator, unless it's the same as the /// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one /// number or two. In this case, we will use ";" as the separator. /// /// The string which will be tokenized. /// The IFormatProvider which controls this tokenization. internal TokenizerHelper(string str, IFormatProvider formatProvider) { char numberSeparator = GetNumericListSeparator(formatProvider); Initialize(str, '\'', numberSeparator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. internal TokenizerHelper(string str, char quoteChar, char separator) { Initialize(str, quoteChar, separator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. private void Initialize(string str, char quoteChar, char separator) { _str = str; _strLen = str == null ? 0 : str.Length; _currentTokenIndex = -1; _quoteChar = quoteChar; _argSeparator = separator; // immediately forward past any whitespace so // NextToken() logic always starts on the first // character of the next token. while (_charIndex < _strLen) { if (!Char.IsWhiteSpace(_str, _charIndex)) { break; } ++_charIndex; } } internal string GetCurrentToken() { // if no current token, return null if (_currentTokenIndex < 0) { return null; } return _str.Substring(_currentTokenIndex,_currentTokenLength); } ////// Throws an exception if there is any non-whitespace left un-parsed. /// internal void LastTokenRequired() { if (_charIndex != _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken() { return NextToken(false); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired() { if (!NextToken(false)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired(bool allowQuotedToken) { if (!NextToken(allowQuotedToken)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken) { // use the currently-set separator character. return NextToken(allowQuotedToken, _argSeparator); } ////// Advances to the NextToken. A separator character can be specified /// which overrides the one previously set. /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken, char separator) { _currentTokenIndex = -1; // reset the currentTokenIndex _foundSeparator = false; // reset // If we're at end of the string, just return false. if (_charIndex >= _strLen) { return false; } char currentChar = _str[_charIndex]; Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace"); // setup the quoteCount int quoteCount = 0; // If we are allowing a quoted token and this token begins with a quote, // set up the quote count and skip the initial quote if (allowQuotedToken && currentChar == _quoteChar) { quoteCount++; // increment quote count ++_charIndex; // move to next character } int newTokenIndex = _charIndex; int newTokenLength = 0; // loop until hit end of string or hit a , or whitespace // if at end of string ust return false. while (_charIndex < _strLen) { currentChar = _str[_charIndex]; // if have a QuoteCount and this is a quote // decrement the quoteCount if (quoteCount > 0) { // if anything but a quoteChar we move on if (currentChar == _quoteChar) { --quoteCount; // if at zero which it always should for now // break out of the loop if (0 == quoteCount) { ++_charIndex; // move past the quote break; } } } else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator)) { if (currentChar == separator) { _foundSeparator = true; } break; } ++_charIndex; ++newTokenLength; } // if quoteCount isn't zero we hit the end of the string // before the ending quote if (quoteCount > 0) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote)); } ScanToNextToken(separator); // move so at the start of the nextToken for next call // finally made it, update the _currentToken values _currentTokenIndex = newTokenIndex; _currentTokenLength = newTokenLength; if (_currentTokenLength < 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } return true; } // helper to move the _charIndex to the next token or to the end of the string void ScanToNextToken(char separator) { // if already at end of the string don't bother if (_charIndex < _strLen) { char currentChar = _str[_charIndex]; // check that the currentChar is a space or the separator. If not // we have an error. this can happen in the quote case // that the char after the quotes string isn't a char. if (!(currentChar == separator) && !Char.IsWhiteSpace(currentChar)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } // loop until hit a character that isn't // an argument separator or whitespace. // !!!Todo: if more than one argSet throw an exception int argSepCount = 0; while (_charIndex < _strLen) { currentChar = _str[_charIndex]; if (currentChar == separator) { _foundSeparator = true; ++argSepCount; _charIndex++; if (argSepCount > 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } else if (Char.IsWhiteSpace(currentChar)) { ++_charIndex; } else { break; } } // if there was a separatorChar then we shouldn't be // at the end of string or means there was a separator // but there isn't an arg if (argSepCount > 0 && _charIndex >= _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } } // Helper to get the numeric list separator for a given IFormatProvider. // Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise. static internal char GetNumericListSeparator(IFormatProvider provider) { char numericSeparator = ','; // Get the NumberFormatInfo out of the provider, if possible // If the IFormatProvider doesn't not contain a NumberFormatInfo, then // this method returns the current culture's NumberFormatInfo. NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider); Debug.Assert(null != numberFormat); // Is the decimal separator is the same as the list separator? // If so, we use the ";". if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0])) { numericSeparator = ';'; } return numericSeparator; } internal bool FoundSeparator { get { return _foundSeparator; } } char _quoteChar; char _argSeparator; string _str; int _strLen; int _charIndex; internal int _currentTokenIndex; internal int _currentTokenLength; bool _foundSeparator; } } // File provided for Reference Use Only by Microsoft Corporation (c) 2007. // Copyright (c) Microsoft Corporation. All rights reserved.
Link Menu
This book is available now!
Buy at Amazon US or
Buy at Amazon UK
- TypographyProperties.cs
- TableParagraph.cs
- DateTimeValueSerializerContext.cs
- FlowDocumentPage.cs
- BooleanSwitch.cs
- CodeTypeReference.cs
- Operand.cs
- FontInfo.cs
- HttpWebResponse.cs
- UnauthorizedAccessException.cs
- WpfXamlType.cs
- ProfileParameter.cs
- MgmtConfigurationRecord.cs
- Substitution.cs
- XmlMembersMapping.cs
- DelegateHelpers.Generated.cs
- MailFileEditor.cs
- ScrollBar.cs
- Hex.cs
- WindowsStatusBar.cs
- XamlParser.cs
- PackWebResponse.cs
- Margins.cs
- ZipIOCentralDirectoryFileHeader.cs
- DeclarativeCatalogPart.cs
- XMLUtil.cs
- EnvelopedPkcs7.cs
- WebColorConverter.cs
- DbBuffer.cs
- coordinator.cs
- StackBuilderSink.cs
- RelationshipEndCollection.cs
- OrderingQueryOperator.cs
- Tuple.cs
- HtmlInputControl.cs
- DefaultClaimSet.cs
- AssemblyInfo.cs
- TypeSystemProvider.cs
- SemaphoreSecurity.cs
- XAMLParseException.cs
- ValidatingPropertiesEventArgs.cs
- GridItemCollection.cs
- WebServiceResponseDesigner.cs
- CodeDomExtensionMethods.cs
- ObjectItemConventionAssemblyLoader.cs
- Mappings.cs
- WebAdminConfigurationHelper.cs
- RequestQueryProcessor.cs
- COM2TypeInfoProcessor.cs
- TdsParser.cs
- PageHandlerFactory.cs
- EnumType.cs
- ChannelAcceptor.cs
- CompilerGeneratedAttribute.cs
- RuleRef.cs
- NumberSubstitution.cs
- SortDescriptionCollection.cs
- Documentation.cs
- BooleanToVisibilityConverter.cs
- IPipelineRuntime.cs
- securitycriticaldataformultiplegetandset.cs
- IfAction.cs
- TransactedReceiveData.cs
- printdlgexmarshaler.cs
- ProgressChangedEventArgs.cs
- SAPICategories.cs
- ResourcePermissionBaseEntry.cs
- DataGridHeaderBorder.cs
- AdornerLayer.cs
- HtmlSelect.cs
- UserPreferenceChangingEventArgs.cs
- ToolstripProfessionalRenderer.cs
- Configuration.cs
- CodeDomLoader.cs
- SerialReceived.cs
- GZipStream.cs
- ThreadInterruptedException.cs
- Vector3DAnimationBase.cs
- TextEditorContextMenu.cs
- RestHandlerFactory.cs
- NameTable.cs
- FocusWithinProperty.cs
- ContainerAction.cs
- HtmlTableRowCollection.cs
- TransformCryptoHandle.cs
- FlowLayoutPanel.cs
- ControlIdConverter.cs
- HashCodeCombiner.cs
- WebDisplayNameAttribute.cs
- ExpressionBinding.cs
- Single.cs
- Avt.cs
- ProxyHwnd.cs
- LinkLabel.cs
- EntityDataSourceUtil.cs
- SessionEndingEventArgs.cs
- AppSecurityManager.cs
- XmlParserContext.cs
- ResourcesBuildProvider.cs
- DataGridViewCellStyle.cs