Code:
/ Net / Net / 3.5.50727.3053 / DEVDIV / depot / DevDiv / releases / Orcas / SP / wpf / src / Shared / MS / Internal / TokenizerHelper.cs / 1 / TokenizerHelper.cs
//---------------------------------------------------------------------------- // // Copyright (C) Microsoft Corporation. All rights reserved. // // File: TokenizerHelper.cs // // Description: This file contains the implementation of TokenizerHelper. // This class should be used by most - if not all - MIL parsers. // // History: // 05/19/2003 : adsmith - Created it // 05/20/2003 : adsmith - Moved to Shared // //--------------------------------------------------------------------------- using System; using System.Collections; using System.Diagnostics; using System.ComponentModel; using System.Globalization; #if !PBTCOMPILER using System.Windows; #endif #if WINDOWS_BASE using MS.Internal.WindowsBase; #elif PRESENTATION_CORE using MS.Internal.PresentationCore; #elif PRESENTATIONFRAMEWORK using MS.Internal.PresentationFramework; #elif PBTCOMPILER using MS.Utility ; using MS.Internal.Markup; #elif DRT using MS.Internal.Drt; #else #error Attempt to use FriendAccessAllowedAttribute from an unknown assembly. using MS.Internal.YourAssemblyName; #endif #if PBTCOMPILER namespace MS.Internal.Markup #else namespace MS.Internal #endif { #if !PBTCOMPILER [FriendAccessAllowed] #endif internal class TokenizerHelper { ////// Constructor for TokenizerHelper which accepts an IFormatProvider. /// If the IFormatProvider is null, we use the thread's IFormatProvider info. /// We will use ',' as the list separator, unless it's the same as the /// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one /// number or two. In this case, we will use ";" as the separator. /// /// The string which will be tokenized. /// The IFormatProvider which controls this tokenization. internal TokenizerHelper(string str, IFormatProvider formatProvider) { char numberSeparator = GetNumericListSeparator(formatProvider); Initialize(str, '\'', numberSeparator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. internal TokenizerHelper(string str, char quoteChar, char separator) { Initialize(str, quoteChar, separator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. private void Initialize(string str, char quoteChar, char separator) { _str = str; _strLen = str == null ? 0 : str.Length; _currentTokenIndex = -1; _quoteChar = quoteChar; _argSeparator = separator; // immediately forward past any whitespace so // NextToken() logic always starts on the first // character of the next token. while (_charIndex < _strLen) { if (!Char.IsWhiteSpace(_str, _charIndex)) { break; } ++_charIndex; } } internal string GetCurrentToken() { // if no current token, return null if (_currentTokenIndex < 0) { return null; } return _str.Substring(_currentTokenIndex,_currentTokenLength); } ////// Throws an exception if there is any non-whitespace left un-parsed. /// internal void LastTokenRequired() { if (_charIndex != _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken() { return NextToken(false); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired() { if (!NextToken(false)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired(bool allowQuotedToken) { if (!NextToken(allowQuotedToken)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken) { // use the currently-set separator character. return NextToken(allowQuotedToken, _argSeparator); } ////// Advances to the NextToken. A separator character can be specified /// which overrides the one previously set. /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken, char separator) { _currentTokenIndex = -1; // reset the currentTokenIndex _foundSeparator = false; // reset // If we're at end of the string, just return false. if (_charIndex >= _strLen) { return false; } char currentChar = _str[_charIndex]; Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace"); // setup the quoteCount int quoteCount = 0; // If we are allowing a quoted token and this token begins with a quote, // set up the quote count and skip the initial quote if (allowQuotedToken && currentChar == _quoteChar) { quoteCount++; // increment quote count ++_charIndex; // move to next character } int newTokenIndex = _charIndex; int newTokenLength = 0; // loop until hit end of string or hit a , or whitespace // if at end of string ust return false. while (_charIndex < _strLen) { currentChar = _str[_charIndex]; // if have a QuoteCount and this is a quote // decrement the quoteCount if (quoteCount > 0) { // if anything but a quoteChar we move on if (currentChar == _quoteChar) { --quoteCount; // if at zero which it always should for now // break out of the loop if (0 == quoteCount) { ++_charIndex; // move past the quote break; } } } else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator)) { if (currentChar == separator) { _foundSeparator = true; } break; } ++_charIndex; ++newTokenLength; } // if quoteCount isn't zero we hit the end of the string // before the ending quote if (quoteCount > 0) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote)); } ScanToNextToken(separator); // move so at the start of the nextToken for next call // finally made it, update the _currentToken values _currentTokenIndex = newTokenIndex; _currentTokenLength = newTokenLength; if (_currentTokenLength < 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } return true; } // helper to move the _charIndex to the next token or to the end of the string void ScanToNextToken(char separator) { // if already at end of the string don't bother if (_charIndex < _strLen) { char currentChar = _str[_charIndex]; // check that the currentChar is a space or the separator. If not // we have an error. this can happen in the quote case // that the char after the quotes string isn't a char. if (!(currentChar == separator) && !Char.IsWhiteSpace(currentChar)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } // loop until hit a character that isn't // an argument separator or whitespace. // !!!Todo: if more than one argSet throw an exception int argSepCount = 0; while (_charIndex < _strLen) { currentChar = _str[_charIndex]; if (currentChar == separator) { _foundSeparator = true; ++argSepCount; _charIndex++; if (argSepCount > 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } else if (Char.IsWhiteSpace(currentChar)) { ++_charIndex; } else { break; } } // if there was a separatorChar then we shouldn't be // at the end of string or means there was a separator // but there isn't an arg if (argSepCount > 0 && _charIndex >= _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } } // Helper to get the numeric list separator for a given IFormatProvider. // Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise. static internal char GetNumericListSeparator(IFormatProvider provider) { char numericSeparator = ','; // Get the NumberFormatInfo out of the provider, if possible // If the IFormatProvider doesn't not contain a NumberFormatInfo, then // this method returns the current culture's NumberFormatInfo. NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider); Debug.Assert(null != numberFormat); // Is the decimal separator is the same as the list separator? // If so, we use the ";". if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0])) { numericSeparator = ';'; } return numericSeparator; } internal bool FoundSeparator { get { return _foundSeparator; } } char _quoteChar; char _argSeparator; string _str; int _strLen; int _charIndex; internal int _currentTokenIndex; internal int _currentTokenLength; bool _foundSeparator; } } // File provided for Reference Use Only by Microsoft Corporation (c) 2007. // Copyright (c) Microsoft Corporation. All rights reserved. //---------------------------------------------------------------------------- // // Copyright (C) Microsoft Corporation. All rights reserved. // // File: TokenizerHelper.cs // // Description: This file contains the implementation of TokenizerHelper. // This class should be used by most - if not all - MIL parsers. // // History: // 05/19/2003 : adsmith - Created it // 05/20/2003 : adsmith - Moved to Shared // //--------------------------------------------------------------------------- using System; using System.Collections; using System.Diagnostics; using System.ComponentModel; using System.Globalization; #if !PBTCOMPILER using System.Windows; #endif #if WINDOWS_BASE using MS.Internal.WindowsBase; #elif PRESENTATION_CORE using MS.Internal.PresentationCore; #elif PRESENTATIONFRAMEWORK using MS.Internal.PresentationFramework; #elif PBTCOMPILER using MS.Utility ; using MS.Internal.Markup; #elif DRT using MS.Internal.Drt; #else #error Attempt to use FriendAccessAllowedAttribute from an unknown assembly. using MS.Internal.YourAssemblyName; #endif #if PBTCOMPILER namespace MS.Internal.Markup #else namespace MS.Internal #endif { #if !PBTCOMPILER [FriendAccessAllowed] #endif internal class TokenizerHelper { ////// Constructor for TokenizerHelper which accepts an IFormatProvider. /// If the IFormatProvider is null, we use the thread's IFormatProvider info. /// We will use ',' as the list separator, unless it's the same as the /// decimal separator. If it *is*, then we can't determine if, say, "23,5" is one /// number or two. In this case, we will use ";" as the separator. /// /// The string which will be tokenized. /// The IFormatProvider which controls this tokenization. internal TokenizerHelper(string str, IFormatProvider formatProvider) { char numberSeparator = GetNumericListSeparator(formatProvider); Initialize(str, '\'', numberSeparator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. internal TokenizerHelper(string str, char quoteChar, char separator) { Initialize(str, quoteChar, separator); } ////// Initialize the TokenizerHelper with the string to tokenize, /// the char which represents quotes and the list separator. /// /// The string to tokenize. /// The quote char. /// The list separator. private void Initialize(string str, char quoteChar, char separator) { _str = str; _strLen = str == null ? 0 : str.Length; _currentTokenIndex = -1; _quoteChar = quoteChar; _argSeparator = separator; // immediately forward past any whitespace so // NextToken() logic always starts on the first // character of the next token. while (_charIndex < _strLen) { if (!Char.IsWhiteSpace(_str, _charIndex)) { break; } ++_charIndex; } } internal string GetCurrentToken() { // if no current token, return null if (_currentTokenIndex < 0) { return null; } return _str.Substring(_currentTokenIndex,_currentTokenLength); } ////// Throws an exception if there is any non-whitespace left un-parsed. /// internal void LastTokenRequired() { if (_charIndex != _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken() { return NextToken(false); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired() { if (!NextToken(false)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken, throwing an exception if not present /// ///The next token found internal string NextTokenRequired(bool allowQuotedToken) { if (!NextToken(allowQuotedToken)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperPrematureStringTermination)); } return GetCurrentToken(); } ////// Advances to the NextToken /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken) { // use the currently-set separator character. return NextToken(allowQuotedToken, _argSeparator); } ////// Advances to the NextToken. A separator character can be specified /// which overrides the one previously set. /// ///true if next token was found, false if at end of string internal bool NextToken(bool allowQuotedToken, char separator) { _currentTokenIndex = -1; // reset the currentTokenIndex _foundSeparator = false; // reset // If we're at end of the string, just return false. if (_charIndex >= _strLen) { return false; } char currentChar = _str[_charIndex]; Debug.Assert(!Char.IsWhiteSpace(currentChar),"Token started on Whitespace"); // setup the quoteCount int quoteCount = 0; // If we are allowing a quoted token and this token begins with a quote, // set up the quote count and skip the initial quote if (allowQuotedToken && currentChar == _quoteChar) { quoteCount++; // increment quote count ++_charIndex; // move to next character } int newTokenIndex = _charIndex; int newTokenLength = 0; // loop until hit end of string or hit a , or whitespace // if at end of string ust return false. while (_charIndex < _strLen) { currentChar = _str[_charIndex]; // if have a QuoteCount and this is a quote // decrement the quoteCount if (quoteCount > 0) { // if anything but a quoteChar we move on if (currentChar == _quoteChar) { --quoteCount; // if at zero which it always should for now // break out of the loop if (0 == quoteCount) { ++_charIndex; // move past the quote break; } } } else if ((Char.IsWhiteSpace(currentChar)) || (currentChar == separator)) { if (currentChar == separator) { _foundSeparator = true; } break; } ++_charIndex; ++newTokenLength; } // if quoteCount isn't zero we hit the end of the string // before the ending quote if (quoteCount > 0) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperMissingEndQuote)); } ScanToNextToken(separator); // move so at the start of the nextToken for next call // finally made it, update the _currentToken values _currentTokenIndex = newTokenIndex; _currentTokenLength = newTokenLength; if (_currentTokenLength < 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } return true; } // helper to move the _charIndex to the next token or to the end of the string void ScanToNextToken(char separator) { // if already at end of the string don't bother if (_charIndex < _strLen) { char currentChar = _str[_charIndex]; // check that the currentChar is a space or the separator. If not // we have an error. this can happen in the quote case // that the char after the quotes string isn't a char. if (!(currentChar == separator) && !Char.IsWhiteSpace(currentChar)) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperExtraDataEncountered)); } // loop until hit a character that isn't // an argument separator or whitespace. // !!!Todo: if more than one argSet throw an exception int argSepCount = 0; while (_charIndex < _strLen) { currentChar = _str[_charIndex]; if (currentChar == separator) { _foundSeparator = true; ++argSepCount; _charIndex++; if (argSepCount > 1) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } else if (Char.IsWhiteSpace(currentChar)) { ++_charIndex; } else { break; } } // if there was a separatorChar then we shouldn't be // at the end of string or means there was a separator // but there isn't an arg if (argSepCount > 0 && _charIndex >= _strLen) { throw new System.InvalidOperationException(SR.Get(SRID.TokenizerHelperEmptyToken)); } } } // Helper to get the numeric list separator for a given IFormatProvider. // Separator is a comma [,] if the decimal separator is not a comma, or a semicolon [;] otherwise. static internal char GetNumericListSeparator(IFormatProvider provider) { char numericSeparator = ','; // Get the NumberFormatInfo out of the provider, if possible // If the IFormatProvider doesn't not contain a NumberFormatInfo, then // this method returns the current culture's NumberFormatInfo. NumberFormatInfo numberFormat = NumberFormatInfo.GetInstance(provider); Debug.Assert(null != numberFormat); // Is the decimal separator is the same as the list separator? // If so, we use the ";". if ((numberFormat.NumberDecimalSeparator.Length > 0) && (numericSeparator == numberFormat.NumberDecimalSeparator[0])) { numericSeparator = ';'; } return numericSeparator; } internal bool FoundSeparator { get { return _foundSeparator; } } char _quoteChar; char _argSeparator; string _str; int _strLen; int _charIndex; internal int _currentTokenIndex; internal int _currentTokenLength; bool _foundSeparator; } } // File provided for Reference Use Only by Microsoft Corporation (c) 2007. // Copyright (c) Microsoft Corporation. All rights reserved.
Link Menu

This book is available now!
Buy at Amazon US or
Buy at Amazon UK
- BamlResourceSerializer.cs
- FaultHandlingFilter.cs
- SmiXetterAccessMap.cs
- Serializer.cs
- RegexCapture.cs
- storepermissionattribute.cs
- ZipIOBlockManager.cs
- HtmlEncodedRawTextWriter.cs
- TextCompositionManager.cs
- SafeTokenHandle.cs
- HitTestResult.cs
- DetailsViewPagerRow.cs
- DesignerLoader.cs
- DataIdProcessor.cs
- XamlToRtfWriter.cs
- SqlException.cs
- OpenTypeMethods.cs
- ToolStripItemTextRenderEventArgs.cs
- DataSourceExpression.cs
- TileBrush.cs
- WebBaseEventKeyComparer.cs
- Trace.cs
- ValidationEventArgs.cs
- EarlyBoundInfo.cs
- UnsafeNativeMethods.cs
- XpsS0ValidatingLoader.cs
- VirtualDirectoryMappingCollection.cs
- PrintDialog.cs
- Clipboard.cs
- DataGridPageChangedEventArgs.cs
- BufferedResponseStream.cs
- mansign.cs
- SqlCacheDependency.cs
- SerializerWriterEventHandlers.cs
- FrameworkElementAutomationPeer.cs
- InternalsVisibleToAttribute.cs
- IdnElement.cs
- JournalEntryStack.cs
- DataGridViewRowsRemovedEventArgs.cs
- BitmapScalingModeValidation.cs
- IArgumentProvider.cs
- PagePropertiesChangingEventArgs.cs
- ResXResourceSet.cs
- LinqToSqlWrapper.cs
- ManifestSignedXml.cs
- ExpressionBuilder.cs
- DataGridViewDataConnection.cs
- HelpInfo.cs
- ImageList.cs
- RequestQueryProcessor.cs
- OleServicesContext.cs
- TriggerBase.cs
- TextBoxRenderer.cs
- PreloadedPackages.cs
- SqlWriter.cs
- _ReceiveMessageOverlappedAsyncResult.cs
- DefaultPropertyAttribute.cs
- HttpListenerContext.cs
- SrgsDocumentParser.cs
- Collection.cs
- NavigationProperty.cs
- Marshal.cs
- XmlSchemaAnyAttribute.cs
- PingReply.cs
- EntityDataSourceValidationException.cs
- ValidationPropertyAttribute.cs
- ProvidersHelper.cs
- UnicodeEncoding.cs
- ListDictionaryInternal.cs
- SmtpAuthenticationManager.cs
- AlphabetConverter.cs
- EntityTypeEmitter.cs
- EntityContainerRelationshipSet.cs
- SparseMemoryStream.cs
- MimeTypePropertyAttribute.cs
- ResXResourceSet.cs
- AspNetSynchronizationContext.cs
- BaseValidator.cs
- RegexNode.cs
- QueueAccessMode.cs
- Size.cs
- DataGridToolTip.cs
- DesignerVerbCollection.cs
- ValueTypeFieldReference.cs
- IndexerHelper.cs
- FormsAuthenticationUserCollection.cs
- StackOverflowException.cs
- ReadOnlyNameValueCollection.cs
- DetailsViewInsertEventArgs.cs
- PackageRelationship.cs
- UpdateTranslator.cs
- CachedTypeface.cs
- DataGridViewLayoutData.cs
- VBIdentifierName.cs
- OleDbPermission.cs
- ActiveDesignSurfaceEvent.cs
- ListenerChannelContext.cs
- FloaterParagraph.cs
- TraceListeners.cs
- CommonDialog.cs